$ oc get lokistack lokistack-hub -oyaml
apiVersion: loki.grafana.com/v1
kind: LokiStack
metadata:
  annotations:
    loki.grafana.com/rulesDiscoveredAt: "2023-12-27T01:21:39Z"
  creationTimestamp: "2023-12-27T01:20:44Z"
  generation: 1
  name: lokistack-hub
  namespace: openshift-logging
  resourceVersion: "73608"
  uid: 7f1581fc-17f7-43cb-bc09-f2e3018d156a
spec:
  managementState: Managed
  size: 1x.demo
  storage:
    schemas:
    - effectiveDate: "2020-10-11"
      version: v11
    secret:
      name: hub-lokistack-s3-credentials
      type: s3
  storageClassName: gp3-csi
  tenants:
    authentication:
    - mTLS:
        ca:
          caName: qitang-1227-1
      tenantId: qitang-1227-1
      tenantName: qitang-1227-1
    - mTLS:
        ca:
          caName: qitang-1227-2
      tenantId: qitang-1227-2
      tenantName: qitang-1227-2
    authorization:
      roleBindings:
      - name: write-logs
        roles:
        - write-logs
        subjects:
        - kind: group
          name: logging-ocm-addon
      - name: read-logs
        roles:
        - read-logs
        subjects:
        - kind: group
          name: logging-ocm-addon
      roles:
      - name: read-logs
        permissions:
        - read
        resources:
        - logs
        tenants:
        - qitang-1227-1
        - qitang-1227-2
      - name: write-logs
        permissions:
        - write
        resources:
        - logs
        tenants:
        - qitang-1227-1
        - qitang-1227-2
    mode: static
status:
  components: {}
  conditions:
  - lastTransitionTime: "2023-12-27T01:24:53Z"
    message: Missing configmap for tenant qitang-1227-1
    reason: MissingGatewayTenantConfigMap
    status: "True"
    type: Degraded
  storage: {} 

$ oc get cm -n openshift-logging
NAME                       DATA   AGE
kube-root-ca.crt           1      59m
lokistack-hub-ca-bundle    1      9m57s
openshift-service-ca.crt   1      59m
qitang-1227-2              1      7m44s 

spokeClusters:
  - qitang-1227-1
  - qitang-1227-2


lokiURL: https://lokistack-hub-openshift-logging.apps.xxxx.openshift.com


# Create a ConfigMap in openshift-logging with the CA bundle, this field should
# match the field with the same name under the multi-cluster-logging Chart
certManagerCerts: true 

$ cat demo/multi-cluster-logging/values.yaml 
# list of spoke clusters that will send logs to Loki
spokeClusters:
  - qitang-1227-1
  - qitang-1227-2


# Loki S3 bucket configuration
lokiS3Bucket:
  endpoint: https://s3.us-east-2.amazonaws.com
  region: us-east-2
  accessKeyID: xxxx
  accessKeySecret: xxxx
  bucketnames: logging-loki-qitang


# Enable & install cert-manager, to issue&manage client certificates for mTLS
# communication for each spoke cluster
certManagerCerts: true 

$ oc logs -n open-cluster-management                            logging-addon-manager-controller-546bd667c5-znkpj
W1227 01:21:39.596682       1 client_config.go:618] Neither --kubeconfig nor --master was specified.  Using the inClusterConfig.  This might not work.
I1227 01:21:40.232133       1 serving.go:342] Generated self-signed cert (/tmp/serving-cert2680212014/tls.crt, /tmp/serving-cert2680212014/tls.key)
W1227 01:21:40.672184       1 authorization.go:47] Authorization is disabled
W1227 01:21:40.672218       1 authentication.go:49] Authentication is disabled
I1227 01:21:40.675871       1 dynamic_serving_content.go:132] "Starting controller" name="serving-cert::/tmp/serving-cert2680212014/tls.crt::/tmp/serving-cert2680212014/tls.key"
I1227 01:21:40.676975       1 secure_serving.go:210] Serving securely on [::]:8443
I1227 01:21:40.677051       1 tlsconfig.go:240] "Starting DynamicServingCertificateController"
I1227 01:21:40.703591       1 base_controller.go:34] Waiting for caches to sync for CSRApprovingController
I1227 01:21:40.703611       1 base_controller.go:34] Waiting for caches to sync for addon-owner-controller
I1227 01:21:40.703653       1 base_controller.go:34] Waiting for caches to sync for addon-config-controller
I1227 01:21:40.703653       1 base_controller.go:34] Waiting for caches to sync for addon-deploy-controller
I1227 01:21:40.703669       1 base_controller.go:34] Waiting for caches to sync for addon-registration-controller
I1227 01:21:40.703676       1 base_controller.go:34] Waiting for caches to sync for management-addon-config-controller
I1227 01:21:40.703690       1 base_controller.go:34] Waiting for caches to sync for addon-configuration-controller
I1227 01:21:40.703704       1 base_controller.go:34] Waiting for caches to sync for CSRApprovingController
I1227 01:21:40.703733       1 base_controller.go:34] Waiting for caches to sync for addon-install-controller
I1227 01:21:40.803864       1 base_controller.go:40] Caches are synced for addon-install-controller 
I1227 01:21:40.803886       1 base_controller.go:78] Starting #1 worker of addon-install-controller controller ...
I1227 01:21:40.803886       1 base_controller.go:40] Caches are synced for addon-registration-controller 
I1227 01:21:40.803902       1 base_controller.go:78] Starting #1 worker of addon-registration-controller controller ...
I1227 01:21:40.803866       1 base_controller.go:40] Caches are synced for addon-owner-controller 
I1227 01:21:40.803924       1 base_controller.go:40] Caches are synced for addon-deploy-controller 
I1227 01:21:40.803931       1 base_controller.go:78] Starting #1 worker of addon-owner-controller controller ...
I1227 01:21:40.803940       1 base_controller.go:40] Caches are synced for CSRApprovingController 
I1227 01:21:40.803948       1 base_controller.go:40] Caches are synced for addon-configuration-controller 
I1227 01:21:40.803956       1 base_controller.go:78] Starting #1 worker of addon-configuration-controller controller ...
I1227 01:21:40.803939       1 base_controller.go:78] Starting #1 worker of addon-deploy-controller controller ...
I1227 01:21:40.803949       1 base_controller.go:78] Starting #1 worker of CSRApprovingController controller ...
I1227 01:21:40.803924       1 base_controller.go:40] Caches are synced for CSRApprovingController 
I1227 01:21:40.803983       1 base_controller.go:78] Starting #1 worker of CSRApprovingController controller ...
I1227 01:21:41.603995       1 base_controller.go:40] Caches are synced for addon-config-controller 
I1227 01:21:41.604014       1 base_controller.go:78] Starting #1 worker of addon-config-controller controller ...
I1227 01:21:41.604055       1 base_controller.go:40] Caches are synced for management-addon-config-controller 
I1227 01:21:41.604075       1 base_controller.go:78] Starting #1 worker of management-addon-config-controller controller ...
E1227 01:23:00.931514       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: Operation cannot be fulfilled on managedclusteraddons.addon.open-cluster-management.io "logging-ocm-addon": the object has been modified; please apply your changes to the latest version and try again
E1227 01:23:00.933697       1 base_controller.go:159] "addon-owner-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: Operation cannot be fulfilled on managedclusteraddons.addon.open-cluster-management.io "logging-ocm-addon": the object has been modified; please apply your changes to the latest version and try again
E1227 01:23:00.944404       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: Operation cannot be fulfilled on managedclusteraddons.addon.open-cluster-management.io "logging-ocm-addon": the object has been modified; please apply your changes to the latest version and try again
E1227 01:23:00.944702       1 base_controller.go:159] "addon-owner-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: Operation cannot be fulfilled on managedclusteraddons.addon.open-cluster-management.io "logging-ocm-addon": the object has been modified; please apply your changes to the latest version and try again
E1227 01:23:00.966194       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:00.972150       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:00.982396       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:01.000340       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:23:01.000741       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:01.003012       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:01.024240       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
I1227 01:23:01.079045       1 csr_helpers.go:174] CSR approved
E1227 01:23:01.083872       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:01.124488       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:01.244236       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:01.373951       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:23:01.885397       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:02.174735       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:23:02.975062       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:23:03.166256       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:03.774809       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:23:04.574429       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:23:05.374271       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:23:05.726669       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:06.175607       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:23:06.974687       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:23:07.774220       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:23:10.847571       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:12.921950       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:23:21.088440       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:21.514235       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:23:21.537475       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:23:23.187670       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:23:41.568966       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:24:04.175210       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:24:53.635200       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:25:03.489684       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:25:26.123222       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:28:09.990946       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:30:21.335856       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:30:31.169986       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:31:40.706449       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:31:40.709658       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:31:40.714104       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:31:40.739499       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found
E1227 01:31:41.559873       1 base_controller.go:159] "addon-config-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: object has no spec field
E1227 01:33:37.697414       1 base_controller.go:159] "addon-deploy-controller" controller failed to sync "qitang-1227-2/logging-ocm-addon", err: configmaps "lokistack-hub-gateway-ca-bundle" not found 

$ oc -n cert-manager logs -c cert-manager-controller cert-manager-6668f46b6-2mcrs
I1227 01:22:21.181384       1 start.go:75] "cert-manager: starting controller" version="canary" git-commit=""
I1227 01:22:21.181452       1 controller.go:250] "cert-manager/controller/build-context: configured acme dns01 nameservers" nameservers=["172.30.0.10:53"]
W1227 01:22:21.181505       1 client_config.go:618] Neither --kubeconfig nor --master was specified.  Using the inClusterConfig.  This might not work.
I1227 01:22:21.182422       1 controller.go:71] "cert-manager/controller: enabled controllers: [certificaterequests-approver certificaterequests-issuer-acme certificaterequests-issuer-ca certificaterequests-issuer-selfsigned certificaterequests-issuer-vault certificaterequests-issuer-venafi certificates-issuing certificates-key-manager certificates-metrics certificates-readiness certificates-request-manager certificates-revision-manager certificates-trigger challenges clusterissuers ingress-shim issuers orders]"
I1227 01:22:21.182843       1 controller.go:144] "cert-manager/controller: starting leader election"
I1227 01:22:21.182871       1 controller.go:137] "cert-manager/controller: starting healthz server" address="[::]:9403"
I1227 01:22:21.182887       1 controller.go:92] "cert-manager/controller: starting metrics server" address="[::]:9402"
I1227 01:22:21.183565       1 leaderelection.go:245] attempting to acquire leader lease kube-system/cert-manager-controller...
I1227 01:22:21.209002       1 leaderelection.go:255] successfully acquired lease kube-system/cert-manager-controller
I1227 01:22:21.210001       1 controller.go:191] "cert-manager/controller: not starting controller as it's disabled" controller="certificatesigningrequests-issuer-acme"
I1227 01:22:21.210118       1 controller.go:191] "cert-manager/controller: not starting controller as it's disabled" controller="certificatesigningrequests-issuer-vault"
I1227 01:22:21.210626       1 controller.go:214] "cert-manager/controller: starting controller" controller="certificates-revision-manager"
I1227 01:22:21.211030       1 controller.go:191] "cert-manager/controller: not starting controller as it's disabled" controller="gateway-shim"
I1227 01:22:21.211157       1 controller.go:214] "cert-manager/controller: starting controller" controller="clusterissuers"
I1227 01:22:21.211392       1 controller.go:214] "cert-manager/controller: starting controller" controller="certificates-readiness"
I1227 01:22:21.211717       1 controller.go:214] "cert-manager/controller: starting controller" controller="certificates-trigger"
I1227 01:22:21.212084       1 controller.go:214] "cert-manager/controller: starting controller" controller="orders"
I1227 01:22:21.212346       1 controller.go:214] "cert-manager/controller: starting controller" controller="ingress-shim"
I1227 01:22:21.213022       1 controller.go:214] "cert-manager/controller: starting controller" controller="certificaterequests-issuer-selfsigned"
I1227 01:22:21.213349       1 controller.go:214] "cert-manager/controller: starting controller" controller="certificaterequests-issuer-venafi"
I1227 01:22:21.213616       1 controller.go:214] "cert-manager/controller: starting controller" controller="certificates-issuing"
I1227 01:22:21.213917       1 controller.go:214] "cert-manager/controller: starting controller" controller="certificates-metrics"
I1227 01:22:21.214076       1 controller.go:191] "cert-manager/controller: not starting controller as it's disabled" controller="certificatesigningrequests-issuer-ca"
I1227 01:22:21.214131       1 controller.go:214] "cert-manager/controller: starting controller" controller="certificates-request-manager"
I1227 01:22:21.214467       1 controller.go:214] "cert-manager/controller: starting controller" controller="challenges"
I1227 01:22:21.215173       1 controller.go:214] "cert-manager/controller: starting controller" controller="certificaterequests-approver"
I1227 01:22:21.215284       1 controller.go:214] "cert-manager/controller: starting controller" controller="certificaterequests-issuer-acme"
I1227 01:22:21.215620       1 controller.go:214] "cert-manager/controller: starting controller" controller="certificaterequests-issuer-ca"
I1227 01:22:21.216014       1 controller.go:214] "cert-manager/controller: starting controller" controller="certificaterequests-issuer-vault"
I1227 01:22:21.216263       1 controller.go:191] "cert-manager/controller: not starting controller as it's disabled" controller="certificatesigningrequests-issuer-selfsigned"
I1227 01:22:21.216276       1 controller.go:191] "cert-manager/controller: not starting controller as it's disabled" controller="certificatesigningrequests-issuer-venafi"
I1227 01:22:21.216357       1 controller.go:214] "cert-manager/controller: starting controller" controller="certificates-key-manager"
I1227 01:22:21.217593       1 controller.go:214] "cert-manager/controller: starting controller" controller="issuers"
I1227 01:22:21.818043       1 conditions.go:96] Setting lastTransitionTime for Issuer "bootstrap-issuer" condition "Ready" to 2023-12-27 01:22:21.818033587 +0000 UTC m=+0.661012927
I1227 01:22:21.911891       1 trigger_controller.go:194] "cert-manager/certificates-trigger: Certificate must be re-issued" key="qitang-1227-2/qitang-1227-2" reason="DoesNotExist" message="Issuing certificate as Secret does not exist"
I1227 01:22:21.911892       1 trigger_controller.go:194] "cert-manager/certificates-trigger: Certificate must be re-issued" key="cert-manager/root-certificate" reason="DoesNotExist" message="Issuing certificate as Secret does not exist"
I1227 01:22:21.911908       1 trigger_controller.go:194] "cert-manager/certificates-trigger: Certificate must be re-issued" key="qitang-1227-1/qitang-1227-1" reason="DoesNotExist" message="Issuing certificate as Secret does not exist"
I1227 01:22:21.911946       1 conditions.go:203] Setting lastTransitionTime for Certificate "root-certificate" condition "Issuing" to 2023-12-27 01:22:21.911940491 +0000 UTC m=+0.754919841
I1227 01:22:21.911935       1 conditions.go:203] Setting lastTransitionTime for Certificate "qitang-1227-2" condition "Issuing" to 2023-12-27 01:22:21.911928396 +0000 UTC m=+0.754907747
I1227 01:22:21.911958       1 conditions.go:203] Setting lastTransitionTime for Certificate "qitang-1227-1" condition "Issuing" to 2023-12-27 01:22:21.911952786 +0000 UTC m=+0.754932136
E1227 01:22:21.912031       1 setup.go:48] "cert-manager/clusterissuers/setup: error getting signing CA TLS certificate" err="secret \"root-certificate\" not found" resource_name="root-issuer" resource_namespace="" resource_kind="ClusterIssuer" resource_version="v1"
I1227 01:22:21.912101       1 conditions.go:96] Setting lastTransitionTime for Issuer "root-issuer" condition "Ready" to 2023-12-27 01:22:21.912096136 +0000 UTC m=+0.755075476
E1227 01:22:21.912122       1 sync.go:62] "cert-manager/clusterissuers: error setting up issuer" err="secret \"root-certificate\" not found" resource_name="root-issuer" resource_namespace="" resource_kind="ClusterIssuer" resource_version="v1"
I1227 01:22:21.912132       1 conditions.go:203] Setting lastTransitionTime for Certificate "qitang-1227-1" condition "Ready" to 2023-12-27 01:22:21.912120945 +0000 UTC m=+0.755100297
I1227 01:22:21.912247       1 conditions.go:203] Setting lastTransitionTime for Certificate "root-certificate" condition "Ready" to 2023-12-27 01:22:21.912242437 +0000 UTC m=+0.755221787
I1227 01:22:21.912321       1 conditions.go:203] Setting lastTransitionTime for Certificate "qitang-1227-2" condition "Ready" to 2023-12-27 01:22:21.912316756 +0000 UTC m=+0.755296106
E1227 01:22:21.927678       1 controller.go:167] "cert-manager/clusterissuers: re-queuing item due to error processing" err="secret \"root-certificate\" not found" key="root-issuer"
E1227 01:22:21.927828       1 setup.go:48] "cert-manager/clusterissuers/setup: error getting signing CA TLS certificate" err="secret \"root-certificate\" not found" resource_name="root-issuer" resource_namespace="" resource_kind="ClusterIssuer" resource_version="v1"
E1227 01:22:21.927898       1 sync.go:62] "cert-manager/clusterissuers: error setting up issuer" err="secret \"root-certificate\" not found" resource_name="root-issuer" resource_namespace="" resource_kind="ClusterIssuer" resource_version="v1"
E1227 01:22:21.927931       1 controller.go:167] "cert-manager/clusterissuers: re-queuing item due to error processing" err="secret \"root-certificate\" not found" key="root-issuer"
I1227 01:22:21.934906       1 controller.go:162] "cert-manager/certificates-readiness: re-queuing item due to optimistic locking on resource" key="qitang-1227-2/qitang-1227-2" error="Operation cannot be fulfilled on certificates.cert-manager.io \"qitang-1227-2\": the object has been modified; please apply your changes to the latest version and try again"
I1227 01:22:21.934957       1 conditions.go:203] Setting lastTransitionTime for Certificate "qitang-1227-2" condition "Ready" to 2023-12-27 01:22:21.934951525 +0000 UTC m=+0.777930868
I1227 01:22:21.937783       1 controller.go:162] "cert-manager/certificates-trigger: re-queuing item due to optimistic locking on resource" key="cert-manager/root-certificate" error="Operation cannot be fulfilled on certificates.cert-manager.io \"root-certificate\": the object has been modified; please apply your changes to the latest version and try again"
I1227 01:22:21.937900       1 trigger_controller.go:194] "cert-manager/certificates-trigger: Certificate must be re-issued" key="cert-manager/root-certificate" reason="DoesNotExist" message="Issuing certificate as Secret does not exist"
I1227 01:22:21.937936       1 conditions.go:203] Setting lastTransitionTime for Certificate "root-certificate" condition "Issuing" to 2023-12-27 01:22:21.937931435 +0000 UTC m=+0.780910776
I1227 01:22:21.941294       1 controller.go:162] "cert-manager/certificates-readiness: re-queuing item due to optimistic locking on resource" key="qitang-1227-1/qitang-1227-1" error="Operation cannot be fulfilled on certificates.cert-manager.io \"qitang-1227-1\": the object has been modified; please apply your changes to the latest version and try again"
I1227 01:22:21.941348       1 conditions.go:203] Setting lastTransitionTime for Certificate "qitang-1227-1" condition "Ready" to 2023-12-27 01:22:21.941341988 +0000 UTC m=+0.784321338
I1227 01:22:23.130009       1 controller.go:162] "cert-manager/certificates-key-manager: re-queuing item due to optimistic locking on resource" key="qitang-1227-2/qitang-1227-2" error="Operation cannot be fulfilled on certificates.cert-manager.io \"qitang-1227-2\": the object has been modified; please apply your changes to the latest version and try again"
I1227 01:22:23.183751       1 conditions.go:263] Setting lastTransitionTime for CertificateRequest "qitang-1227-2-j6xmc" condition "Approved" to 2023-12-27 01:22:23.18374074 +0000 UTC m=+2.026720087
I1227 01:22:23.216054       1 conditions.go:263] Setting lastTransitionTime for CertificateRequest "qitang-1227-2-j6xmc" condition "Ready" to 2023-12-27 01:22:23.216032647 +0000 UTC m=+2.059011988
I1227 01:22:23.226314       1 controller.go:162] "cert-manager/certificates-key-manager: re-queuing item due to optimistic locking on resource" key="qitang-1227-1/qitang-1227-1" error="Operation cannot be fulfilled on certificates.cert-manager.io \"qitang-1227-1\": the object has been modified; please apply your changes to the latest version and try again"
I1227 01:22:23.265108       1 conditions.go:263] Setting lastTransitionTime for CertificateRequest "qitang-1227-1-b2qdt" condition "Approved" to 2023-12-27 01:22:23.265097861 +0000 UTC m=+2.108077213
I1227 01:22:23.288001       1 conditions.go:263] Setting lastTransitionTime for CertificateRequest "qitang-1227-1-b2qdt" condition "Ready" to 2023-12-27 01:22:23.287983353 +0000 UTC m=+2.130962695
I1227 01:22:23.550440       1 conditions.go:263] Setting lastTransitionTime for CertificateRequest "root-certificate-dfb8c" condition "Approved" to 2023-12-27 01:22:23.550429658 +0000 UTC m=+2.393409009
I1227 01:22:23.595260       1 conditions.go:263] Setting lastTransitionTime for CertificateRequest "root-certificate-dfb8c" condition "Ready" to 2023-12-27 01:22:23.595252064 +0000 UTC m=+2.438231405
I1227 01:22:23.625078       1 conditions.go:192] Found status change for Certificate "root-certificate" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-12-27 01:22:23.625072538 +0000 UTC m=+2.468051869
I1227 01:22:23.639286       1 controller.go:162] "cert-manager/certificates-readiness: re-queuing item due to optimistic locking on resource" key="cert-manager/root-certificate" error="Operation cannot be fulfilled on certificates.cert-manager.io \"root-certificate\": the object has been modified; please apply your changes to the latest version and try again"
I1227 01:22:23.640624       1 conditions.go:192] Found status change for Certificate "root-certificate" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-12-27 01:22:23.640620092 +0000 UTC m=+2.483599419
I1227 01:22:23.659902       1 controller.go:162] "cert-manager/certificates-key-manager: re-queuing item due to optimistic locking on resource" key="cert-manager/root-certificate" error="Operation cannot be fulfilled on certificates.cert-manager.io \"root-certificate\": the object has been modified; please apply your changes to the latest version and try again"
I1227 01:22:26.929274       1 conditions.go:85] Found status change for Issuer "root-issuer" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-12-27 01:22:26.92926689 +0000 UTC m=+5.772246231
I1227 01:22:26.957429       1 conditions.go:252] Found status change for CertificateRequest "qitang-1227-1-b2qdt" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-12-27 01:22:26.957421511 +0000 UTC m=+5.800400852
I1227 01:22:26.958137       1 conditions.go:252] Found status change for CertificateRequest "qitang-1227-2-j6xmc" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-12-27 01:22:26.958129112 +0000 UTC m=+5.801108452
I1227 01:22:26.994404       1 conditions.go:192] Found status change for Certificate "qitang-1227-2" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-12-27 01:22:26.994397886 +0000 UTC m=+5.837377249
I1227 01:22:26.994987       1 conditions.go:192] Found status change for Certificate "qitang-1227-1" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-12-27 01:22:26.994981197 +0000 UTC m=+5.837960565
I1227 01:22:27.011726       1 controller.go:162] "cert-manager/certificates-readiness: re-queuing item due to optimistic locking on resource" key="qitang-1227-2/qitang-1227-2" error="Operation cannot be fulfilled on certificates.cert-manager.io \"qitang-1227-2\": the object has been modified; please apply your changes to the latest version and try again"
I1227 01:22:27.013105       1 conditions.go:192] Found status change for Certificate "qitang-1227-2" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-12-27 01:22:27.013099614 +0000 UTC m=+5.856078954
I1227 01:22:27.014715       1 controller.go:162] "cert-manager/certificates-readiness: re-queuing item due to optimistic locking on resource" key="qitang-1227-1/qitang-1227-1" error="Operation cannot be fulfilled on certificates.cert-manager.io \"qitang-1227-1\": the object has been modified; please apply your changes to the latest version and try again"
I1227 01:22:27.016197       1 conditions.go:192] Found status change for Certificate "qitang-1227-1" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-12-27 01:22:27.016189957 +0000 UTC m=+5.859169304
I1227 01:22:27.033363       1 controller.go:162] "cert-manager/certificates-readiness: re-queuing item due to optimistic locking on resource" key="qitang-1227-1/qitang-1227-1" error="Operation cannot be fulfilled on certificates.cert-manager.io \"qitang-1227-1\": the object has been modified; please apply your changes to the latest version and try again"
I1227 01:22:27.034184       1 controller.go:162] "cert-manager/certificates-key-manager: re-queuing item due to optimistic locking on resource" key="qitang-1227-2/qitang-1227-2" error="Operation cannot be fulfilled on certificates.cert-manager.io \"qitang-1227-2\": the object has been modified; please apply your changes to the latest version and try again"
I1227 01:22:27.034748       1 conditions.go:192] Found status change for Certificate "qitang-1227-1" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-12-27 01:22:27.034743718 +0000 UTC m=+5.877723052



$ oc -n cert-manager logs -c cert-manager-cainjector cert-manager-cainjector-56bfcf86f8-c229n
I1227 01:21:51.954922       1 start.go:180] "starting" version="canary" revision=""
I1227 01:21:51.977152       1 setup.go:119] "cert-manager: Registering a reconciler for injectable" kind="mutatingwebhookconfiguration"
I1227 01:21:51.978508       1 setup.go:119] "cert-manager: Registering a reconciler for injectable" kind="validatingwebhookconfiguration"
I1227 01:21:51.978567       1 setup.go:119] "cert-manager: Registering a reconciler for injectable" kind="apiservice"
I1227 01:21:51.979924       1 setup.go:119] "cert-manager: Registering a reconciler for injectable" kind="customresourcedefinition"
I1227 01:21:52.380890       1 leaderelection.go:245] attempting to acquire leader lease kube-system/cert-manager-cainjector-leader-election...
I1227 01:21:52.389702       1 leaderelection.go:255] successfully acquired lease kube-system/cert-manager-cainjector-leader-election
I1227 01:21:52.389929       1 recorder.go:104] "cert-manager/events: cert-manager-cainjector-56bfcf86f8-c229n_6afe5a79-e49b-4aca-ac73-9ce2c52fe813 became leader" type="Normal" object={"kind":"Lease","namespace":"kube-system","name":"cert-manager-cainjector-leader-election","uid":"e4814273-f7f7-4450-a663-1d94695c8531","apiVersion":"coordination.k8s.io/v1","resourceVersion":"71129"} reason="LeaderElection"
I1227 01:21:52.389931       1 controller.go:177] "cert-manager: Starting EventSource" controller="mutatingwebhookconfiguration" controllerGroup="admissionregistration.k8s.io" controllerKind="MutatingWebhookConfiguration" source="kind source: *v1.MutatingWebhookConfiguration"
I1227 01:21:52.389952       1 controller.go:177] "cert-manager: Starting EventSource" controller="validatingwebhookconfiguration" controllerGroup="admissionregistration.k8s.io" controllerKind="ValidatingWebhookConfiguration" source="kind source: *v1.ValidatingWebhookConfiguration"
I1227 01:21:52.389970       1 controller.go:177] "cert-manager: Starting EventSource" controller="customresourcedefinition" controllerGroup="apiextensions.k8s.io" controllerKind="CustomResourceDefinition" source="kind source: *v1.CustomResourceDefinition"
I1227 01:21:52.389981       1 controller.go:177] "cert-manager: Starting EventSource" controller="apiservice" controllerGroup="apiregistration.k8s.io" controllerKind="APIService" source="kind source: *v1.APIService"
I1227 01:21:52.389986       1 controller.go:177] "cert-manager: Starting EventSource" controller="customresourcedefinition" controllerGroup="apiextensions.k8s.io" controllerKind="CustomResourceDefinition" source="kind source: *v1.Secret"
I1227 01:21:52.389972       1 controller.go:177] "cert-manager: Starting EventSource" controller="mutatingwebhookconfiguration" controllerGroup="admissionregistration.k8s.io" controllerKind="MutatingWebhookConfiguration" source="kind source: *v1.Secret"
I1227 01:21:52.389997       1 controller.go:177] "cert-manager: Starting EventSource" controller="customresourcedefinition" controllerGroup="apiextensions.k8s.io" controllerKind="CustomResourceDefinition" source="kind source: *v1.Secret"
I1227 01:21:52.389999       1 controller.go:177] "cert-manager: Starting EventSource" controller="apiservice" controllerGroup="apiregistration.k8s.io" controllerKind="APIService" source="kind source: *v1.Secret"
I1227 01:21:52.390010       1 controller.go:177] "cert-manager: Starting EventSource" controller="customresourcedefinition" controllerGroup="apiextensions.k8s.io" controllerKind="CustomResourceDefinition" source="kind source: *v1.Certificate"
I1227 01:21:52.390015       1 controller.go:177] "cert-manager: Starting EventSource" controller="mutatingwebhookconfiguration" controllerGroup="admissionregistration.k8s.io" controllerKind="MutatingWebhookConfiguration" source="kind source: *v1.Secret"
I1227 01:21:52.390020       1 controller.go:185] "cert-manager: Starting Controller" controller="customresourcedefinition" controllerGroup="apiextensions.k8s.io" controllerKind="CustomResourceDefinition"
I1227 01:21:52.390034       1 controller.go:177] "cert-manager: Starting EventSource" controller="mutatingwebhookconfiguration" controllerGroup="admissionregistration.k8s.io" controllerKind="MutatingWebhookConfiguration" source="kind source: *v1.Certificate"
I1227 01:21:52.390048       1 controller.go:185] "cert-manager: Starting Controller" controller="mutatingwebhookconfiguration" controllerGroup="admissionregistration.k8s.io" controllerKind="MutatingWebhookConfiguration"
I1227 01:21:52.390015       1 controller.go:177] "cert-manager: Starting EventSource" controller="apiservice" controllerGroup="apiregistration.k8s.io" controllerKind="APIService" source="kind source: *v1.Secret"
I1227 01:21:52.390098       1 controller.go:177] "cert-manager: Starting EventSource" controller="apiservice" controllerGroup="apiregistration.k8s.io" controllerKind="APIService" source="kind source: *v1.Certificate"
I1227 01:21:52.390109       1 controller.go:185] "cert-manager: Starting Controller" controller="apiservice" controllerGroup="apiregistration.k8s.io" controllerKind="APIService"
I1227 01:21:52.389973       1 controller.go:177] "cert-manager: Starting EventSource" controller="validatingwebhookconfiguration" controllerGroup="admissionregistration.k8s.io" controllerKind="ValidatingWebhookConfiguration" source="kind source: *v1.Secret"
I1227 01:21:52.390128       1 controller.go:177] "cert-manager: Starting EventSource" controller="validatingwebhookconfiguration" controllerGroup="admissionregistration.k8s.io" controllerKind="ValidatingWebhookConfiguration" source="kind source: *v1.Secret"
I1227 01:21:52.390144       1 controller.go:177] "cert-manager: Starting EventSource" controller="validatingwebhookconfiguration" controllerGroup="admissionregistration.k8s.io" controllerKind="ValidatingWebhookConfiguration" source="kind source: *v1.Certificate"
I1227 01:21:52.390158       1 controller.go:185] "cert-manager: Starting Controller" controller="validatingwebhookconfiguration" controllerGroup="admissionregistration.k8s.io" controllerKind="ValidatingWebhookConfiguration"
I1227 01:21:52.604386       1 controller.go:219] "cert-manager: Starting workers" controller="customresourcedefinition" controllerGroup="apiextensions.k8s.io" controllerKind="CustomResourceDefinition" worker count=1
I1227 01:21:52.604409       1 controller.go:219] "cert-manager: Starting workers" controller="validatingwebhookconfiguration" controllerGroup="admissionregistration.k8s.io" controllerKind="ValidatingWebhookConfiguration" worker count=1
I1227 01:21:52.604388       1 controller.go:219] "cert-manager: Starting workers" controller="mutatingwebhookconfiguration" controllerGroup="admissionregistration.k8s.io" controllerKind="MutatingWebhookConfiguration" worker count=1
I1227 01:21:52.604416       1 controller.go:219] "cert-manager: Starting workers" controller="apiservice" controllerGroup="apiregistration.k8s.io" controllerKind="APIService" worker count=1
I1227 01:21:52.611321       1 reconciler.go:142] "cert-manager: Updated object" kind="mutatingwebhookconfiguration" kind="mutatingwebhookconfiguration" name="cert-manager-webhook"
I1227 01:21:52.611989       1 reconciler.go:142] "cert-manager: Updated object" kind="validatingwebhookconfiguration" kind="validatingwebhookconfiguration" name="cert-manager-webhook"
I1227 01:21:52.616709       1 reconciler.go:142] "cert-manager: Updated object" kind="mutatingwebhookconfiguration" kind="mutatingwebhookconfiguration" name="cert-manager-webhook"
I1227 01:21:52.617439       1 reconciler.go:142] "cert-manager: Updated object" kind="validatingwebhookconfiguration" kind="validatingwebhookconfiguration" name="cert-manager-webhook"
I1227 01:21:52.640539       1 reconciler.go:142] "cert-manager: Updated object" kind="customresourcedefinition" kind="customresourcedefinition" name="challenges.acme.cert-manager.io"
I1227 01:21:52.682487       1 reconciler.go:142] "cert-manager: Updated object" kind="customresourcedefinition" kind="customresourcedefinition" name="clusterissuers.cert-manager.io"
I1227 01:21:52.694900       1 reconciler.go:142] "cert-manager: Updated object" kind="customresourcedefinition" kind="customresourcedefinition" name="certificates.cert-manager.io"
I1227 01:21:52.703176       1 reconciler.go:142] "cert-manager: Updated object" kind="customresourcedefinition" kind="customresourcedefinition" name="certificaterequests.cert-manager.io"
I1227 01:21:52.741328       1 reconciler.go:142] "cert-manager: Updated object" kind="customresourcedefinition" kind="customresourcedefinition" name="issuers.cert-manager.io"
I1227 01:21:52.749774       1 reconciler.go:142] "cert-manager: Updated object" kind="customresourcedefinition" kind="customresourcedefinition" name="orders.acme.cert-manager.io"


$ oc -n cert-manager logs -c cert-manager-webhook cert-manager-webhook-867bb5f4f-lcbdd
W1227 01:21:51.971569       1 client_config.go:618] Neither --kubeconfig nor --master was specified.  Using the inClusterConfig.  This might not work.
I1227 01:21:51.995963       1 webhook.go:128] "cert-manager: using dynamic certificate generating using CA stored in Secret resource" secret_namespace="cert-manager" secret_name="cert-manager-webhook-ca"
I1227 01:21:51.996326       1 server.go:133] "cert-manager/webhook: listening for insecure healthz connections" address=":6080"
I1227 01:21:51.996403       1 server.go:197] "cert-manager/webhook: listening for secure connections" address=":10250"
I1227 01:21:53.000645       1 dynamic_source.go:255] "cert-manager/webhook: Updated cert-manager webhook TLS certificate" DNSNames=["cert-manager-webhook","cert-manager-webhook.cert-manager","cert-manager-webhook.cert-manager.svc"]
I1227 01:22:21.919158       1 logs.go:59] http: TLS handshake error from 10.130.0.2:45000: read tcp 10.128.2.59:10250->10.130.0.2:45000: read: connection reset by peer
I1227 01:22:21.920778       1 logs.go:59] http: TLS handshake error from 10.130.0.2:45012: EOF
I1227 01:22:21.922809       1 logs.go:59] http: TLS handshake error from 10.130.0.2:45018: EOF
I1227 01:22:21.924213       1 logs.go:59] http: TLS handshake error from 10.130.0.2:45068: EOF
I1227 01:22:21.924444       1 logs.go:59] http: TLS handshake error from 10.130.0.2:45028: EOF
I1227 01:22:21.925550       1 logs.go:59] http: TLS handshake error from 10.130.0.2:45082: EOF
I1227 01:22:21.931140       1 logs.go:59] http: TLS handshake error from 10.130.0.2:45092: EOF
I1227 01:22:26.970445       1 logs.go:59] http: TLS handshake error from 10.130.0.2:34524: EOF
I1227 01:22:27.002510       1 logs.go:59] http: TLS handshake error from 10.130.0.2:34530: read tcp 10.128.2.59:10250->10.130.0.2:34530: read: connection reset by peer
I1227 01:22:27.004099       1 logs.go:59] http: TLS handshake error from 10.130.0.2:34542: EOF
I1227 01:22:27.005442       1 logs.go:59] http: TLS handshake error from 10.130.0.2:34552: EOF
I1227 01:22:27.022754       1 logs.go:59] http: TLS handshake error from 10.130.0.2:34562: EOF
I1227 01:22:27.026062       1 logs.go:59] http: TLS handshake error from 10.130.0.2:34566: EOF