// quay 3.16.0 final RC build in OCP 4.19
quay.io/redhat-user-workloads/quay-eng-tenant/stable-3-16-v4-19@sha256:b0bee7eb89436350a7dca460f0e9312816cd5cbd34e59d428340534e8658b795

// quay builder image
quay.io/redhat-user-workloads/quay-eng-tenant/quay-builder-v3-16@sha256:f653a5da744696a82ce9f24627a64099a59fc51cbef0134ab3bba3b629ff2c60

FEATURE_BUILD_SUPPORT: true
FEATURE_GITLAB_BUILD: true
GITLAB_TRIGGER_CONFIG:
CLIENT_ID: xxxxxxxxx 
CLIENT_SECRET: xxxxxxxxx 
GITLAB_TRIGGER_KIND:
GITLAB_ENDPOINT: [https://gitlab.com/]
FEATURE_BITBUCKET_BUILD: true
BITBUCKET_TRIGGER_CONFIG:
CONSUMER_KEY: xxxxxxxxx
CONSUMER_SECRET: xxxxxxxxx
FEATURE_GITHUB_BUILD: true
GITHUB_TRIGGER_CONFIG:
API_ENDPOINT: [https://api.github.com/]
CLIENT_ID: xxxxxxxxx
CLIENT_SECRET: xxxxxxxxx
GITHUB_ENDPOINT: [https://github.com/]
BUILDMAN_HOSTNAME: quayregistry-quay-builder-quay-enterprise.apps.whu419az36.qe.azure.devcluster.openshift.com:443 
BUILD_MANAGER:
 - ephemeral
 - ALLOWED_WORKER_COUNT: 20
ORCHESTRATOR_PREFIX: buildman/production/
ORCHESTRATOR:
REDIS_HOST: quayregistry-quay-redis
REDIS_PASSWORD: ""
REDIS_SSL: false
REDIS_SKIP_KEYSPACE_EVENT_SETUP: false
EXECUTORS:
 - EXECUTOR: kubernetesPodman
DEBUG: true
NAME: openshift
BUILDER_NAMESPACE: virtual-builds 
SETUP_TIME: 180
QUAY_USERNAME: xxxxxxxxx
QUAY_PASSWORD: xxxxxxxxx 
BUILDER_CONTAINER_IMAGE: quay.io/redhat-user-workloads/quay-eng-tenant/quay-builder-v3-16@sha256:f653a5da744696a82ce9f24627a64099a59fc51cbef0134ab3bba3b629ff2c60

 # Kubernetes resource options
K8S_API_SERVER: api.whu419az36.qe.azure.devcluster.openshift.com:6443 
K8S_API_TLS_CA: /conf/stack/extra_ca_certs/build_cluster.crt
VOLUME_SIZE: 8G
KUBERNETES_DISTRIBUTION: openshift
CONTAINER_MEMORY_LIMITS: 1G 
CONTAINER_CPU_LIMITS: 1000m
CONTAINER_MEMORY_REQUEST: 1G 
CONTAINER_CPU_REQUEST: 500m
NODE_SELECTOR_LABEL_KEY: ""
NODE_SELECTOR_LABEL_VALUE: ""
SERVICE_ACCOUNT_NAME: quay-builder 
SERVICE_ACCOUNT_TOKEN: xxxxxxxxx
USERFILES_LOCATION: default
USERFILES_PATH: userfiles

% oc get all -n virtual-builds 
Warning: apps.openshift.io/v1 DeploymentConfig is deprecated in v4.14+, unavailable in v4.10000+
NAME READY STATUS RESTARTS AGE
pod/0e0465c8-22c2-4d81-af9e-41f392b09480-bxvzb-49qsx 0/1 Error 0 17m
.......
pod/0e0465c8-22c2-4d81-af9e-41f392b09480-v6zsq-9gpk9 0/1 Error 0 21m

NAME STATUS COMPLETIONS DURATION AGE
job.batch/0e0465c8-22c2-4d81-af9e-41f392b09480-bxvzb Failed 0/1 18m 18m
.......
job.batch/0e0465c8-22c2-4d81-af9e-41f392b09480-v6zsq Failed 0/1 21m 21m

% oc logs 0e0465c8-22c2-4d81-af9e-41f392b09480-v6zsq-9gpk9 -n virtual-builds
[INFO]: CA_CERT found, writing out to /certs/cacert.crt
[INFO]: Waiting for podman to start. Checking again in 3s...
time="2025-12-11T07:29:44Z" level=info msg="podman filtering at log level debug"
time="2025-12-11T07:29:44Z" level=debug msg="Called service.PersistentPreRunE(podman --log-level=debug system service --time 0)"
time="2025-12-11T07:29:44Z" level=info msg="Setting parallel job count to 49"
time="2025-12-11T07:29:44Z" level=debug msg="Using conmon: \"/usr/bin/conmon\""
time="2025-12-11T07:29:44Z" level=info msg="Using sqlite as database backend"
time="2025-12-11T07:29:44Z" level=debug msg="systemd-logind: dial unix /var/run/dbus/system_bus_socket: connect: no such file or directory"
time="2025-12-11T07:29:44Z" level=debug msg="Using graph driver vfs"
time="2025-12-11T07:29:44Z" level=debug msg="Using graph root /home/podman/.local/share/containers/storage"
time="2025-12-11T07:29:44Z" level=debug msg="Using run root /tmp/storage-run-1000/containers"
time="2025-12-11T07:29:44Z" level=debug msg="Using static dir /home/podman/.local/share/containers/storage/libpod"
time="2025-12-11T07:29:44Z" level=debug msg="Using tmp dir /tmp/storage-run-1000/libpod/tmp"
time="2025-12-11T07:29:44Z" level=debug msg="Using volume path /home/podman/.local/share/containers/storage/volumes"
time="2025-12-11T07:29:44Z" level=debug msg="Using transient store: false"
time="2025-12-11T07:29:44Z" level=debug msg="Not configuring container store"
time="2025-12-11T07:29:44Z" level=debug msg="Initializing event backend file"
time="2025-12-11T07:29:44Z" level=debug msg="Configured OCI runtime crun-wasm initialization failed: no valid executable found for OCI runtime crun-wasm: invalid argument"
time="2025-12-11T07:29:44Z" level=debug msg="Configured OCI runtime runc initialization failed: no valid executable found for OCI runtime runc: invalid argument"
time="2025-12-11T07:29:44Z" level=debug msg="Configured OCI runtime kata initialization failed: no valid executable found for OCI runtime kata: invalid argument"
time="2025-12-11T07:29:44Z" level=debug msg="Configured OCI runtime youki initialization failed: no valid executable found for OCI runtime youki: invalid argument"
time="2025-12-11T07:29:44Z" level=debug msg="Configured OCI runtime krun initialization failed: no valid executable found for OCI runtime krun: invalid argument"
time="2025-12-11T07:29:44Z" level=debug msg="Configured OCI runtime ocijail initialization failed: no valid executable found for OCI runtime ocijail: invalid argument"
time="2025-12-11T07:29:44Z" level=debug msg="Configured OCI runtime crun-vm initialization failed: no valid executable found for OCI runtime crun-vm: invalid argument"
time="2025-12-11T07:29:44Z" level=debug msg="Configured OCI runtime runj initialization failed: no valid executable found for OCI runtime runj: invalid argument"
time="2025-12-11T07:29:44Z" level=debug msg="Configured OCI runtime runsc initialization failed: no valid executable found for OCI runtime runsc: invalid argument"
time="2025-12-11T07:29:44Z" level=debug msg="Using OCI runtime \"/usr/bin/crun\""
time="2025-12-11T07:29:45Z" level=info msg="podman filtering at log level debug"
time="2025-12-11T07:29:45Z" level=debug msg="Called service.PersistentPreRunE(podman --log-level=debug system service --time 0)"
time="2025-12-11T07:29:45Z" level=info msg="Setting parallel job count to 49"
time="2025-12-11T07:29:45Z" level=debug msg="Using conmon: \"/usr/bin/conmon\""
time="2025-12-11T07:29:45Z" level=info msg="Using sqlite as database backend"
time="2025-12-11T07:29:45Z" level=debug msg="systemd-logind: dial unix /var/run/dbus/system_bus_socket: connect: no such file or directory"
time="2025-12-11T07:29:45Z" level=debug msg="Using graph driver vfs"
time="2025-12-11T07:29:45Z" level=debug msg="Using graph root /home/podman/.local/share/containers/storage"
time="2025-12-11T07:29:45Z" level=debug msg="Using run root /tmp/storage-run-1000/containers"
time="2025-12-11T07:29:45Z" level=debug msg="Using static dir /home/podman/.local/share/containers/storage/libpod"
time="2025-12-11T07:29:45Z" level=debug msg="Using tmp dir /tmp/storage-run-1000/libpod/tmp"
time="2025-12-11T07:29:45Z" level=debug msg="Using volume path /home/podman/.local/share/containers/storage/volumes"
time="2025-12-11T07:29:45Z" level=debug msg="Using transient store: false"
time="2025-12-11T07:29:45Z" level=debug msg="[graphdriver] trying provided driver \"vfs\""
time="2025-12-11T07:29:45Z" level=debug msg="Initializing event backend file"
time="2025-12-11T07:29:45Z" level=debug msg="Configured OCI runtime ocijail initialization failed: no valid executable found for OCI runtime ocijail: invalid argument"
time="2025-12-11T07:29:45Z" level=debug msg="Configured OCI runtime crun-vm initialization failed: no valid executable found for OCI runtime crun-vm: invalid argument"
time="2025-12-11T07:29:45Z" level=debug msg="Configured OCI runtime crun-wasm initialization failed: no valid executable found for OCI runtime crun-wasm: invalid argument"
time="2025-12-11T07:29:45Z" level=debug msg="Configured OCI runtime kata initialization failed: no valid executable found for OCI runtime kata: invalid argument"
time="2025-12-11T07:29:45Z" level=debug msg="Configured OCI runtime youki initialization failed: no valid executable found for OCI runtime youki: invalid argument"
time="2025-12-11T07:29:45Z" level=debug msg="Configured OCI runtime runc initialization failed: no valid executable found for OCI runtime runc: invalid argument"
time="2025-12-11T07:29:45Z" level=debug msg="Configured OCI runtime runj initialization failed: no valid executable found for OCI runtime runj: invalid argument"
time="2025-12-11T07:29:45Z" level=debug msg="Configured OCI runtime runsc initialization failed: no valid executable found for OCI runtime runsc: invalid argument"
time="2025-12-11T07:29:45Z" level=debug msg="Configured OCI runtime krun initialization failed: no valid executable found for OCI runtime krun: invalid argument"
time="2025-12-11T07:29:45Z" level=debug msg="Using OCI runtime \"/usr/bin/crun\""
time="2025-12-11T07:29:45Z" level=debug msg="Successfully loaded 1 networks"
time="2025-12-11T07:29:45Z" level=debug msg="Initialized SHM lock manager at path /libpod_rootless_lock_1000"
time="2025-12-11T07:29:45Z" level=debug msg="Podman detected system restart - performing state refresh"
time="2025-12-11T07:29:45Z" level=debug msg="Could not move to subcgroup: mkdir /sys/fs/cgroup/init: read-only file system"
time="2025-12-11T07:29:45Z" level=debug msg="Could not move to subcgroup: mkdir /sys/fs/cgroup/init: read-only file system"
time="2025-12-11T07:29:45Z" level=debug msg="registered SIGHUP watcher for config"
time="2025-12-11T07:29:45Z" level=info msg="API service listening on \"/tmp/storage-run-1000/podman/podman.sock\". URI: \"unix:///tmp/storage-run-1000/podman/podman.sock\""
time="2025-12-11T07:29:45Z" level=debug msg="waiting for SIGHUP to reload configuration"
time="2025-12-11T07:29:45Z" level=debug msg="CORS Headers were not set"
time="2025-12-11T07:29:45Z" level=debug msg="API service(s) shutting down, idle for 0s"
time="2025-12-11T07:29:45Z" level=debug msg="API service shutdown request ignored as timeout Duration is UnlimitedService"
[INFO]: Waiting for podman to start. Checking again in 3s...
[INFO]: Waiting for podman to start. Checking again in 3s...
[INFO]: Waiting for podman to start. Checking again in 3s...
[INFO]: Waiting for podman to start. Checking again in 3s...
[ERROR]: podman socket not found, exiting