builder stdout | 2020-12-10 07:30:33,243 [79] [ERROR] [buildman.manager.ephemeral] Exception when scheduling job ebf5fe64-29cd-4975-9fe0-4d7833c04868: 'items'
builder stdout | Traceback (most recent call last):
builder stdout |   File "/quay-registry/buildman/manager/ephemeral.py", line 880, in _work_checker
builder stdout |     schedule_success, retry_timeout = self.schedule(build_id)
builder stdout |   File "/quay-registry/buildman/manager/ephemeral.py", line 586, in schedule
builder stdout |     if self._running_workers() >= allowed_worker_count:
builder stdout |   File "/quay-registry/buildman/manager/ephemeral.py", line 784, in _running_workers
builder stdout |     return sum([x.running_builders_count for x in self._ordered_executors])
builder stdout |   File "/quay-registry/buildman/manager/ephemeral.py", line 784, in <listcomp>
builder stdout |     return sum([x.running_builders_count for x in self._ordered_executors])
builder stdout |   File "/quay-registry/buildman/manager/executor.py", line 405, in running_builders_count
builder stdout |     return len(jobs_list.json()["items"])
builder stdout | KeyError: 'items'

lizhang@lzha-mac quay3.4 % oc get pod quays3-quay-app-84dc775d87-52kfv -o json | jq '.spec.containers[0].image'
"registry.redhat.io/quay/quay@sha256:bb58d111dfd3663281f998e10acb49a150245171f0d5215702a2eb75de2f92a9"

ALLOW_PULLS_WITHOUT_STRICT_LOGGING: false
AUTHENTICATION_TYPE: Database
AVATAR_KIND: local
BUILDLOGS_REDIS:
  host: quays3-quay-redis
  port: 6379
DATABASE_SECRET_KEY: 6CgN5QVnmIYqrUvQx60JyXXcngqbBupV5iuLC9WXVkiffW0mlpZQZnRN9x7sl6lfTGsBoc65NVyS3NLG
DB_CONNECTION_ARGS:
  autorollback: true
  threadlocals: true
DB_URI: postgresql://quays3-quay-database:postgres@quays3-quay-database:5432/quays3-quay-database
DEFAULT_TAG_EXPIRATION: 2w
DISTRIBUTED_STORAGE_CONFIG:
  default:
  - S3Storage
  - host: s3.us-east-2.amazonaws.com
    s3_access_key: ***
    s3_bucket: quay340
    s3_secret_key: ***
    storage_path: /quay20201126
DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS:
- default
DISTRIBUTED_STORAGE_PREFERENCE:
- default
ENTERPRISE_LOGO_URL: /static/img/quay-horizontal-color.svg
FEATURE_ACTION_LOG_ROTATION: false
FEATURE_ANONYMOUS_ACCESS: true
FEATURE_APP_SPECIFIC_TOKENS: true
FEATURE_BITBUCKET_BUILD: false
FEATURE_BLACKLISTED_EMAILS: false
FEATURE_BUILD_SUPPORT: true
FEATURE_CHANGE_TAG_EXPIRATION: true
FEATURE_DIRECT_LOGIN: true
FEATURE_GITHUB_BUILD: true
FEATURE_GITHUB_LOGIN: false
FEATURE_GITLAB_BUILD: false
FEATURE_GOOGLE_LOGIN: false
FEATURE_INVITE_ONLY_USER_CREATION: false
FEATURE_MAILING: false
FEATURE_NONSUPERUSER_TEAM_SYNCING_SETUP: false
FEATURE_PARTIAL_USER_AUTOCOMPLETE: true
FEATURE_PROXY_STORAGE: false
FEATURE_REPO_MIRROR: true
FEATURE_SECURITY_NOTIFICATIONS: true
FEATURE_SECURITY_SCANNER: true
FEATURE_SIGNING: false
FEATURE_STORAGE_REPLICATION: false
FEATURE_TEAM_SYNCING: false
FEATURE_USER_CREATION: true
FEATURE_USER_LAST_ACCESSED: true
FEATURE_USER_LOG_ACCESS: false
FEATURE_USER_METADATA: false
FEATURE_USER_RENAME: false
FEATURE_USERNAME_CONFIRMATION: true
FRESH_LOGIN_TIMEOUT: 10m
GITHUB_LOGIN_CONFIG: {}
GITHUB_TRIGGER_CONFIG:
  API_ENDPOINT: https://api.github.com/
  CLIENT_ID: ***
  CLIENT_SECRET: ***
  GITHUB_ENDPOINT: https://github.com/
GITLAB_TRIGGER_KIND: {}
GPG2_PRIVATE_KEY_FILENAME: signing-private.gpg
GPG2_PUBLIC_KEY_FILENAME: signing-public.gpg
LDAP_ALLOW_INSECURE_FALLBACK: false
LDAP_EMAIL_ATTR: mail
LDAP_UID_ATTR: uid
LDAP_URI: ldap://localhost
LOGS_MODEL: database
LOGS_MODEL_CONFIG: {}
MAIL_DEFAULT_SENDER: support@quay.io
MAIL_PORT: 587
MAIL_USE_AUTH: false
MAIL_USE_TLS: false
PREFERRED_URL_SCHEME: https
REGISTRY_TITLE: Quay
REGISTRY_TITLE_SHORT: Quay
REPO_MIRROR_INTERVAL: 30
REPO_MIRROR_TLS_VERIFY: true
SEARCH_MAX_RESULT_PAGE_COUNT: 10
SEARCH_RESULTS_PER_PAGE: 10
SECRET_KEY: 2rtbkvDztZmSS0Oevt9vk7ktb7WUWsplWwiLvcTphalcxtQoGmioEGS21rSHk4DzT1t1yLbuzsuVyKbf
SECURITY_SCANNER_INDEXING_INTERVAL: 30
SECURITY_SCANNER_V4_ENDPOINT: http://quays3-clair-app:80
SECURITY_SCANNER_V4_NAMESPACE_WHITELIST:
- admin
SECURITY_SCANNER_V4_PSK: dWFCVHFqWW5TWTF5VWZ1QzI1UG91WmJpcjJFRkhtVUc=
SERVER_HOSTNAME: quays3-quay-quay1210.apps.quay-perf-399.perf-testing.devcluster.openshift.com
SETUP_COMPLETE: true
SUPER_USERS:
- quay
- admin
TAG_EXPIRATION_OPTIONS:
- 2w
TEAM_RESYNC_STALE_TIME: 60m
TESTING: false
USER_EVENTS_REDIS:
  host: quays3-quay-redis
  port: 6379
USER_RECOVERY_TOKEN_LIFETIME: 30m
BUILD_MANAGER:
- ephemeral
- ALLOWED_WORKER_COUNT: 10
  ORCHESTRATOR_PREFIX: buildman/production/
  ORCHESTRATOR:
    REDIS_HOST: quays3-quay-redis
  EXECUTORS:
  - EXECUTOR: kubernetes
    NAME: openshift
    QUAY_USERNAME: quay
    QUAY_PASSWORD: password
    WORKER_IMAGE: brew.registry.redhat.io/rh-osbs/quay-quay-builder
    WORKER_TAG: v3.4.0-9
    SETUP_TIME: 180
    MINIMUM_RETRY_THRESHOLD: 1
    BUILDER_VM_CONTAINER_IMAGE: brew.registry.redhat.io/rh-osbs/quay-quay-builder-qemu-rhcos:v3.4.0-4
    VERIFY_TLS: false
    K8S_API_SERVER: api.quay-perf-399.perf-testing.devcluster.openshift.com:6443
    VOLUME_SIZE: 8G
    KUBERNETES_DISTRIBUTION: openshift
    CONTAINER_MEMORY_LIMITS: 5120Mi
    CONTAINER_CPU_LIMITS: 1000m
    #CONTAINER_MEMORY_REQUEST: 3968Mi
    #CONTAINER_CPU_REQUEST: 500m
    NODE_SELECTOR_LABEL_KEY: node.kubernetes.io/instance-type
    NODE_SELECTOR_LABEL_VALUE: m4.2xlarge
    SERVICE_ACCOUNT_NAME: kubeadmin
    SERVICE_ACCOUNT_TOKEN: WdApy-qDZax-wxI3v-D2KbI