Details
-
Feature Request
-
Resolution: Unresolved
-
Undefined
-
None
-
None
-
False
-
None
-
False
-
Not Selected
-
x86_64
-
0
-
0%
Description
- Podman push-ing, skopeo copy-ing a 13 gb image to Hitachi HCL s3 storage fails with below error:
FATA[0875] writing blob: uploading layer chunked: received unexpected HTTP status: 500 Internal Server Error
- Quay logs show following python traceback:
gunicorn-registry stdout | 2022-09-21 13:31:16,104 [264] [DEBUG] [botocore.retryhandler] retry needed, retryable exception caught: Read timeout on endpoint URL: "https://redhatopenshiftprod.hyd-hcpg10-116.icicibankltd.com:443/redhatopenshiftprodns/redhat-openshift-prod/uploads/539e85f6-188f-41da-a24e-3bf8569a6f68?up loadId=106481037497217" gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 421, in _make_request gunicorn-registry stdout | six.raise_from(e, None) gunicorn-registry stdout | File "<string>", line 3, in raise_from gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 416, in _make_request gunicorn-registry stdout | httplib_response = conn.getresponse() gunicorn-registry stdout | File "/usr/lib64/python3.8/http/client.py", line 1347, in getresponse gunicorn-registry stdout | response.begin() gunicorn-registry stdout | File "/usr/lib64/python3.8/http/client.py", line 307, in begin gunicorn-registry stdout | version, status, reason = self._read_status() gunicorn-registry stdout | File "/usr/lib64/python3.8/http/client.py", line 268, in _read_status gunicorn-registry stdout | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") gunicorn-registry stdout | File "/usr/lib64/python3.8/socket.py", line 669, in readinto gunicorn-registry stdout | return self._sock.recv_into(b) gunicorn-registry stdout | File "/usr/local/lib64/python3.8/site-packages/gevent/_ssl3.py", line 503, in recv_into gunicorn-registry stdout | return self.read(nbytes, buffer) gunicorn-registry stdout | File "/usr/local/lib64/python3.8/site-packages/gevent/_ssl3.py", line 332, in read gunicorn-registry stdout | self._wait(self._read_event, timeout_exc=_SSLErrorReadTimeout) gunicorn-registry stdout | File "src/gevent/_hub_primitives.py", line 284, in gevent.__hub_primitives.wait_on_socket gunicorn-registry stdout | File "src/gevent/_hub_primitives.py", line 289, in gevent.__hub_primitives.wait_on_socket gunicorn-registry stdout | File "src/gevent/_hub_primitives.py", line 280, in gevent.__hub_primitives._primitive_wait gunicorn-registry stdout | File "src/gevent/_hub_primitives.py", line 281, in gevent.__hub_primitives._primitive_wait gunicorn-registry stdout | File "src/gevent/_hub_primitives.py", line 46, in gevent.__hub_primitives.WaitOperationsGreenlet.wait gunicorn-registry stdout | File "src/gevent/_hub_primitives.py", line 46, in gevent.__hub_primitives.WaitOperationsGreenlet.wait gunicorn-registry stdout | File "src/gevent/_hub_primitives.py", line 55, in gevent.__hub_primitives.WaitOperationsGreenlet.wait gunicorn-registry stdout | File "src/gevent/_waiter.py", line 151, in gevent.__waiter.Waiter.get gunicorn-registry stdout | File "src/gevent/_greenlet_primitives.py", line 60, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch gunicorn-registry stdout | File "src/gevent/_greenlet_primitives.py", line 60, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch gunicorn-registry stdout | File "src/gevent/_greenlet_primitives.py", line 64, in gevent.__greenlet_primitives.SwitchOutGreenletWithLoop.switch gunicorn-registry stdout | File "src/gevent/__greenlet_primitives.pxd", line 35, in gevent.__greenlet_primitives._greenlet_switch gunicorn-registry stdout | socket.timeout: The read operation timed out gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/botocore/httpsession.py", line 311, in send gunicorn-registry stdout | urllib_response = conn.urlopen( gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 719, in urlopen gunicorn-registry stdout | retries = retries.increment( gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/urllib3/util/retry.py", line 376, in increment gunicorn-registry stdout | raise six.reraise(type(error), error, _stacktrace) gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/urllib3/packages/six.py", line 735, in reraise gunicorn-registry stdout | raise value gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 665, in urlopen gunicorn-registry stdout | httplib_response = self._make_request( gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 423, in _make_request gunicorn-registry stdout | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py", line 330, in _raise_timeout gunicorn-registry stdout | raise ReadTimeoutError( gunicorn-registry stdout | urllib3.exceptions.ReadTimeoutError: AWSHTTPSConnectionPool(host='redhatopenshiftprod.hyd-hcpg10-116.icicibankltd.com', port=443): Read timed out. (read timeout=60) gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/botocore/retryhandler.py", line 269, in _should_retry gunicorn-registry stdout | return self._checker(attempt_number, response, caught_exception) gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/botocore/retryhandler.py", line 316, in __call__ gunicorn-registry stdout | checker_response = checker(attempt_number, response, gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/botocore/retryhandler.py", line 222, in __call__ gunicorn-registry stdout | return self._check_caught_exception( gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/botocore/retryhandler.py", line 359, in _check_caught_exception gunicorn-registry stdout | raise caught_exception gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/botocore/endpoint.py", line 200, in _do_get_response gunicorn-registry stdout | http_response = self._send(request) gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/botocore/endpoint.py", line 269, in _send gunicorn-registry stdout | return self.http_session.send(request) gunicorn-registry stdout | File "/usr/local/lib/python3.8/site-packages/botocore/httpsession.py", line 346, in send gunicorn-registry stdout | raise ReadTimeoutError(endpoint_url=request.url, error=e) gunicorn-registry stdout | botocore.exceptions.ReadTimeoutError: Read timeout on endpoint URL: "https://redhatopenshiftprod.hyd-hcpg10-116.icicibankltd.com:443/redhatopenshiftprodns/redhat-openshift-prod/uploads/539e85f6-188f-41da-a24e-3bf8569a6f68?uploadId=106481037497217"
- Directly copying big layers to s3 backend using aws cli commands and read timeout options[1] and then podman pushing is helpful workaround for now but having a parameter in quay will help make the push experience better for the clients. The cli-read-timeout is for "The maximum amount of time (in seconds) to wait between consecutive read operations for a response from the server.
[1] https://docs.aws.amazon.com/cli/latest/reference/
