I am trying to setup quay with an s3 bucket.  The S3 bucket is a self hosted instance of minio reachable under `s3.mydomain.com`.    During config i set `s3.mydomain.com`. And as soon as i tried to push the first image onto the registry the layers retried until failure.    From the logs i extracted    ``` ... gunicorn-registry stdout | 2024-07-01 09:05:11,573 [211] [INFO] [gunicorn.access] - - [01/Jul/2024:09:05:11 +0000] "POST /v2/foerdeareal/mqtt_replay/blobs/uploads/ HTTP/1.1" 500 0 " " " " nginx stdout | 172.26.0.1 - - [01/Jul/2024:09:05:11 +0000] "POST /v2/foerdeareal/mqtt_replay/blobs/uploads/ HTTP/1.1" 500 141 "-" "docker/26.1.4 go/go1.21.11 git-commit/de5c9cf kernel/6.6.31-linuxkit os/linux arch/amd64 UpstreamClient(Docker-Client/26.1.4 \x5C(darwin\x5C))" (6.837 1618 6.836) gunicorn-registry stdout | 2024-07-01 09:05:12,041 [211] [ERROR] [gunicorn.error] Error handling request /v2/foerdeareal/mqtt_replay/blobs/uploads/ gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/urllib3/connection.py", line 174, in _new_conn gunicorn-registry stdout | conn = connection.create_connection( gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/urllib3/util/connection.py", line 72, in create_connection gunicorn-registry stdout | for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/gevent/_socketcommon.py", line 247, in getaddrinfo gunicorn-registry stdout | addrlist = get_hub().resolver.getaddrinfo(host, port, family, type, proto, flags) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/gevent/resolver/thread.py", line 63, in getaddrinfo gunicorn-registry stdout | return self.pool.apply(_socket.getaddrinfo, args, kwargs) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/gevent/pool.py", line 161, in apply gunicorn-registry stdout | return self.spawn(func, *args, **kwds).get() gunicorn-registry stdout | File "src/gevent/event.py", line 329, in gevent._gevent_cevent.AsyncResult.get gunicorn-registry stdout | File "src/gevent/event.py", line 359, in gevent._gevent_cevent.AsyncResult.get gunicorn-registry stdout | File "src/gevent/event.py", line 347, in gevent._gevent_cevent.AsyncResult.get gunicorn-registry stdout | File "src/gevent/event.py", line 327, in gevent._gevent_cevent.AsyncResult._raise_exception gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/gevent/_compat.py", line 65, in reraise gunicorn-registry stdout | raise value.with_traceback(tb) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/gevent/threadpool.py", line 167, in __run_task gunicorn-registry stdout | thread_result.set(func(*args, **kwargs)) gunicorn-registry stdout | socket.gaierror: [Errno -2] Name or service not known gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/httpsession.py", line 439, in send gunicorn-registry stdout | urllib_response = conn.urlopen( gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/urllib3/connectionpool.py", line 785, in urlopen gunicorn-registry stdout | retries = retries.increment( gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/urllib3/util/retry.py", line 525, in increment gunicorn-registry stdout | raise six.reraise(type(error), error, _stacktrace) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/urllib3/packages/six.py", line 770, in reraise gunicorn-registry stdout | raise value gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/urllib3/connectionpool.py", line 703, in urlopen gunicorn-registry stdout | httplib_response = self._make_request( gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/urllib3/connectionpool.py", line 386, in _make_request gunicorn-registry stdout | self._validate_conn(conn) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/urllib3/connectionpool.py", line 1040, in _validate_conn gunicorn-registry stdout | conn.connect() gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/urllib3/connection.py", line 358, in connect gunicorn-registry stdout | self.sock = conn = self._new_conn() gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/urllib3/connection.py", line 186, in _new_conn gunicorn-registry stdout | raise NewConnectionError( gunicorn-registry stdout | urllib3.exceptions.NewConnectionError: <botocore.awsrequest.AWSHTTPSConnection object at 0x77a052cdf8b0>: Failed to establish a new connection: [Errno -2] Name or service not known gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/gunicorn/workers/base_async.py", line 55, in handle gunicorn-registry stdout | self.handle_request(listener_name, req, client, addr) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/gunicorn/workers/ggevent.py", line 127, in handle_request gunicorn-registry stdout | super().handle_request(listener_name, req, sock, addr) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/gunicorn/workers/base_async.py", line 108, in handle_request gunicorn-registry stdout | respiter = self.wsgi(environ, resp.start_response) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/flask/app.py", line 2463, in _ call _ gunicorn-registry stdout | return self.wsgi_app(environ, start_response) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/werkzeug/middleware/proxy_fix.py", line 169, in _ call _ gunicorn-registry stdout | return self.app(environ, start_response) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/flask/app.py", line 2449, in wsgi_app gunicorn-registry stdout | response = self.handle_exception(e) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/flask/app.py", line 1866, in handle_exception gunicorn-registry stdout | reraise(exc_type, exc_value, tb) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/flask/_compat.py", line 39, in reraise gunicorn-registry stdout | raise value gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/flask/app.py", line 2446, in wsgi_app gunicorn-registry stdout | response = self.full_dispatch_request() gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/flask/app.py", line 1951, in full_dispatch_request gunicorn-registry stdout | rv = self.handle_user_exception(e) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/flask/app.py", line 1820, in handle_user_exception gunicorn-registry stdout | reraise(exc_type, exc_value, tb) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/flask/_compat.py", line 39, in reraise gunicorn-registry stdout | raise value gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/flask/app.py", line 1949, in full_dispatch_request gunicorn-registry stdout | rv = self.dispatch_request() gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/flask/app.py", line 1935, in dispatch_request gunicorn-registry stdout | return self.view_functions [rule.endpoint] (**req.view_args) gunicorn-registry stdout | File "/quay-registry/endpoints/decorators.py", line 227, in wrapper gunicorn-registry stdout | return func(*args, **kwargs) gunicorn-registry stdout | File "/quay-registry/endpoints/decorators.py", line 90, in wrapper gunicorn-registry stdout | return func(*args, **kwargs) gunicorn-registry stdout | File "/quay-registry/auth/registry_jwt_auth.py", line 175, in wrapper gunicorn-registry stdout | return func(*args, **kwargs) gunicorn-registry stdout | File "/quay-registry/endpoints/v2/_ init _.py", line 169, in wrapped gunicorn-registry stdout | return func(namespace_name, repo_name, *args, **kwargs) gunicorn-registry stdout | File "/quay-registry/endpoints/decorators.py", line 163, in wrapper gunicorn-registry stdout | return func(*args, **kwargs) gunicorn-registry stdout | File "/quay-registry/endpoints/decorators.py", line 188, in wrapper gunicorn-registry stdout | return func(*args, **kwargs) gunicorn-registry stdout | File "/quay-registry/endpoints/v2/blob.py", line 268, in start_blob_upload gunicorn-registry stdout | blob_uploader = create_blob_upload(repository_ref, storage, _upload_settings()) gunicorn-registry stdout | File "/quay-registry/data/registry_model/blobuploader.py", line 74, in create_blob_upload gunicorn-registry stdout | new_upload_uuid, upload_metadata = storage.initiate_chunked_upload(location_name) gunicorn-registry stdout | File "/quay-registry/storage/distributedstorage.py", line 26, in wrapper gunicorn-registry stdout | return storage_func(*args, **kwargs) gunicorn-registry stdout | File "/quay-registry/storage/cloud.py", line 511, in initiate_chunked_upload gunicorn-registry stdout | self._initialize_cloud_conn() gunicorn-registry stdout | File "/quay-registry/storage/cloud.py", line 149, in _initialize_cloud_conn gunicorn-registry stdout | self._cloud_conn.head_bucket(Bucket=self._bucket_name) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/client.py", line 415, in _api_call gunicorn-registry stdout | return self._make_api_call(operation_name, kwargs) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/client.py", line 731, in _make_api_call gunicorn-registry stdout | http, parsed_response = self._make_request( gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/client.py", line 751, in _make_request gunicorn-registry stdout | return self._endpoint.make_request(operation_model, request_dict) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/endpoint.py", line 107, in make_request gunicorn-registry stdout | return self._send_request(request_dict, operation_model) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/endpoint.py", line 183, in _send_request gunicorn-registry stdout | while self._needs_retry(attempts, operation_model, request_dict, gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/endpoint.py", line 305, in _needs_retry gunicorn-registry stdout | responses = self._event_emitter.emit( gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/hooks.py", line 358, in emit gunicorn-registry stdout | return self._emitter.emit(aliased_event_name, **kwargs) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/hooks.py", line 229, in emit gunicorn-registry stdout | return self._emit(event_name, kwargs) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/hooks.py", line 212, in _emit gunicorn-registry stdout | response = handler(**kwargs) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/retryhandler.py", line 194, in _ call _ gunicorn-registry stdout | if self._checker(**checker_kwargs): gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/retryhandler.py", line 267, in _ call _ gunicorn-registry stdout | should_retry = self._should_retry(attempt_number, response, gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/retryhandler.py", line 294, in _should_retry gunicorn-registry stdout | return self._checker(attempt_number, response, caught_exception) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/retryhandler.py", line 333, in _ call _ gunicorn-registry stdout | checker_response = checker(attempt_number, response, gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/retryhandler.py", line 233, in _ call _ gunicorn-registry stdout | return self._check_caught_exception( gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/retryhandler.py", line 376, in _check_caught_exception gunicorn-registry stdout | raise caught_exception gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/endpoint.py", line 249, in _do_get_response gunicorn-registry stdout | http_response = self._send(request) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/endpoint.py", line 321, in _send gunicorn-registry stdout | return self.http_session.send(request) gunicorn-registry stdout | File "/app/lib/python3.9/site-packages/botocore/httpsession.py", line 468, in send gunicorn-registry stdout | raise EndpointConnectionError(endpoint_url=request.url, error=e) gunicorn-registry stdout | botocore.exceptions.EndpointConnectionError: Could not connect to the endpoint URL: "https://s3.cau-west.amazonaws.com/quay-registry-images" gunicorn-registry stdout | 2024-07-01 09:05:12,042 [211] [INFO] [gunicorn.access] - - [01/Jul/2024:09:05:12 +0000] "POST /v2/foerdeareal/mqtt_replay/blobs/uploads/ HTTP/1.1" 500 0 " " " " nginx stdout | 172.26.0.1 - - [01/Jul/2024:09:05:12 +0000] "POST /v2/foerdeareal/mqtt_replay/blobs/uploads/ HTTP/1.1" 500 141 "-" "docker/26.1.4 go/go1.21.11 git-commit/de5c9cf kernel/6.6.31-linuxkit os/linux arch/amd64 UpstreamClient(Docker-Client/26.1.4 \x5C(darwin\x5C))" (4.682 1618 4.680) storagereplication stdout | 2024-07-01 09:05:15,400 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval [0:01:00] , next run at: 2024-07-01 09:06:15 UTC)" (scheduled at 2024-07-01 09:05:15.399927+00:00) storagereplication stdout | 2024-07-01 09:05:15,401 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval [0:01:00] , next run at: 2024-07-01 09:06:15 UTC)" executed successfully notificationworker stdout | 2024-07-01 09:05:19,502 [68] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval [0:00:10] , next run at: 2024-07-01 09:05:29 UTC)" (scheduled at 2024-07-01 09:05:19.501529+00:00) notificationworker stdout | 2024-07-01 09:05:19,521 [68] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval [0:00:10] , next run at: 2024-07-01 09:05:29 UTC)" executed successfully gcworker stdout | 2024-07-01 09:05:21,172 [58] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval [0:00:30] , next run at: 2024-07-01 09:05:51 UTC)" (scheduled at 2024-07-01 09:05:21.171486+00:00) gcworker stdout | 2024-07-01 09:05:21,189 [58] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval [0:00:30] , next run at: 2024-07-01 09:05:51 UTC)" executed successfully buildlogsarchiver stdout | 2024-07-01 09:05:21,226 [53] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval [0:00:30] , next run at: 2024-07-01 09:05:51 UTC)" (scheduled at 2024-07-01 09:05:21.225505+00:00) ```   As i recognise the `cau-west` region from my minio config i am certain that quay connected successfuly at least once.    ```docker-compose version: '3' services: postgres: image: postgres:16.3 ports: ":5432" environment: POSTGRES_DB: quay POSTGRES_USER: quayuser POSTGRES_PASSWORD: mysecretpassword POSTGRES_EXTENSIONS: pg_trgm healthcheck: test: ["CMD-SHELL", "psql -U ${POSTGRES_USER} -d ${POSTGRES_DB} -c 'SELECT 1' || exit 1"] interval: 30s timeout: 10s retries: 3 volumes: quay_postgres_data:/var/lib/postgresql/data redis: image: redis:7 ports: "6379:6379" volumes: quay_redis_data:/data quay: image: quay.io/projectquay/quay:latest ports: "8080:8080" depends_on: postgres redis environment: QUAY_CONFIG_PATH=/conf/stack/config.yaml QUAY_DISTRIBUTED_STORAGE_PREFERENCE=default volumes: ./quay-config:/conf/stack quay_storage:/datastorage quay_tls:/tls volumes: quay_postgres_data: quay_redis_data: quay_storage: quay_tls: ```   ```conf ALLOW_PULLS_WITHOUT_STRICT_LOGGING: true ALLOWED_OCI_ARTIFACT_TYPES: application/vnd.oci.image.config.v1+json: application/vnd.oci.image.layer.v1.tar+zstd application/vnd.sylabs.sif.config.v1+json: application/vnd.sylabs.sif.layer.v1+tar AUTHENTICATION_TYPE: Database AVATAR_KIND: local BUILDLOGS_REDIS: host: redis port: 6379 DATABASE_SECRET_KEY: 3e4c7c3d-308b-4f33-9463-3f689dbbf6f7 DB_CONNECTION_ARGS: {} DB_URI: postgresql://quayuser:mysecretpassword@postgres/quay DEFAULT_TAG_EXPIRATION: 2w DISTRIBUTED_STORAGE_CONFIG: default: S3Storage host: s3.mydomain.com s3_access_key: ---- s3_bucket: quay-registry-images s3_secret_key: ------ storage_path: /datastorage/registry DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS: [] DISTRIBUTED_STORAGE_PREFERENCE: default FEATURE_ACI_CONVERSION: false FEATURE_ACTION_LOG_ROTATION: false FEATURE_ANONYMOUS_ACCESS: true FEATURE_APP_REGISTRY: true FEATURE_APP_SPECIFIC_TOKENS: true FEATURE_BITBUCKET_BUILD: false FEATURE_BLACKLISTED_EMAILS: false FEATURE_BUILD_SUPPORT: false FEATURE_CHANGE_TAG_EXPIRATION: true FEATURE_DIRECT_LOGIN: true FEATURE_EXTENDED_REPOSITORY_NAMES: true FEATURE_FIPS: false FEATURE_GITHUB_BUILD: false FEATURE_GITHUB_LOGIN: false FEATURE_GITLAB_BUILD: false FEATURE_GOOGLE_LOGIN: false FEATURE_INVITE_ONLY_USER_CREATION: false FEATURE_MAILING: false FEATURE_NONSUPERUSER_TEAM_SYNCING_SETUP: false FEATURE_PARTIAL_USER_AUTOCOMPLETE: true FEATURE_PROXY_STORAGE: false FEATURE_REPO_MIRROR: false FEATURE_REQUIRE_TEAM_INVITE: true FEATURE_RESTRICTED_V1_PUSH: true FEATURE_SECURITY_NOTIFICATIONS: false FEATURE_SECURITY_SCANNER: false FEATURE_STORAGE_REPLICATION: true FEATURE_TEAM_SYNCING: false FEATURE_USER_CREATION: true FEATURE_USER_LAST_ACCESSED: true FEATURE_USER_LOG_ACCESS: false FEATURE_USER_METADATA: false FEATURE_USER_RENAME: false FEATURE_USERNAME_CONFIRMATION: true FEATURE_USER_INITIALIZE: true SUPER_USERS: quayadmin FRESH_LOGIN_TIMEOUT: 10m GITHUB_LOGIN_CONFIG: {} GITHUB_TRIGGER_CONFIG: {} GITLAB_TRIGGER_KIND: {} LDAP_ALLOW_INSECURE_FALLBACK: false LDAP_EMAIL_ATTR: mail LDAP_UID_ATTR: uid LDAP_URI: ldap://localhost LOG_ARCHIVE_LOCATION: default LOGS_MODEL: database LOGS_MODEL_CONFIG: {} MAIL_DEFAULT_SENDER: support@quay.io MAIL_PORT: 587 MAIL_USE_AUTH: false MAIL_USE_TLS: false PREFERRED_URL_SCHEME: http REGISTRY_TITLE: ------ REGISTRY_TITLE_SHORT: registry REPO_MIRROR_INTERVAL: 30 REPO_MIRROR_TLS_VERIFY: true SEARCH_MAX_RESULT_PAGE_COUNT: 10 SEARCH_RESULTS_PER_PAGE: 10 SECRET_KEY: ------- SECURITY_SCANNER_INDEXING_INTERVAL: 30 SERVER_HOSTNAME: registry.mydomain.com SETUP_COMPLETE: true TAG_EXPIRATION_OPTIONS: 0s 1d 1w 2w 4w TEAM_RESYNC_STALE_TIME: 30m TESTING: false USE_CDN: false USER_EVENTS_REDIS: host: redis port: 6379 USER_RECOVERY_TOKEN_LIFETIME: 30m USERFILES_LOCATION: minio ```