diff --git a/Makefile b/Makefile index 463c3475..749eaa72 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the BUNDLE_VERSION as arg of the bundle target (e.g make bundle BUNDLE_VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export BUNDLE_VERSION=0.0.2) -BUNDLE_VERSION ?= 1.18.0 +BUNDLE_VERSION ?= $(VERSION) CERT_MANAGER_VERSION ?= "v1.18.2" ISTIO_CSR_VERSION ?= "v0.14.2" @@ -37,7 +37,8 @@ IMAGE_TAG_BASE ?= openshift.io/cert-manager-operator # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) -BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(BUNDLE_VERSION) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) +INDEX_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) # BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command BUNDLE_GEN_FLAGS ?= -q --overwrite=false --version $(BUNDLE_VERSION) $(BUNDLE_METADATA_OPTS) @@ -49,9 +50,9 @@ USE_IMAGE_DIGESTS ?= false ifeq ($(USE_IMAGE_DIGESTS), true) BUNDLE_GEN_FLAGS += --use-image-digests endif - +IMG_VERSION ?= $(VERSION) # Image URL to use all building/pushing image targets -IMG ?= $(IMAGE_TAG_BASE):$(IMG_VERSION) +IMG ?= $(IMAGE_TAG_BASE):v$(IMG_VERSION) # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.25.0 @@ -83,7 +84,7 @@ BIN_DIR=$(shell pwd)/bin SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec -CONTAINER_ENGINE ?= podman +CONTAINER_ENGINE ?= docker CONTAINER_PUSH_ARGS ?= $(if $(filter ${CONTAINER_ENGINE}, docker), , --tls-verify=${TLS_VERIFY}) TLS_VERIFY ?= true CONTAINER_IMAGE_NAME ?= registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.24-openshift-4.20 @@ -213,11 +214,11 @@ run: check-tools manifests generate fmt vet ## Run a controller from your host. .PHONY: image-build image-build: check-tools ## Build container image with the operator. - $(CONTAINER_ENGINE) build -t ${IMG} . + docker build -t ${IMG} . .PHONY: image-push image-push: check-tools ## Push container image with the operator. - $(CONTAINER_ENGINE) push ${IMG} ${CONTAINER_PUSH_ARGS} + docker push ${IMG} ${CONTAINER_PUSH_ARGS} ##@ Deployment @@ -244,11 +245,11 @@ bundle: check-tools $(OPERATOR_SDK_BIN) manifests .PHONY: bundle-image-build bundle-image-build: check-tools bundle - $(CONTAINER_ENGINE) build -t ${BUNDLE_IMG} -f bundle.Dockerfile . + docker build -t ${BUNDLE_IMG} -f bundle.Dockerfile . .PHONY: bundle-image-push bundle-image-push: check-tools - $(CONTAINER_ENGINE) push ${BUNDLE_IMG} + docker push ${BUNDLE_IMG} .PHONY: index-image-build index-image-build: check-tools opm diff --git a/bundle/manifests/cert-manager-operator.clusterserviceversion.yaml b/bundle/manifests/cert-manager-operator.clusterserviceversion.yaml index d8b4c955..77b580fb 100644 --- a/bundle/manifests/cert-manager-operator.clusterserviceversion.yaml +++ b/bundle/manifests/cert-manager-operator.clusterserviceversion.yaml @@ -238,7 +238,7 @@ metadata: capabilities: Seamless Upgrades categories: Security console.openshift.io/disable-operand-delete: "true" - containerImage: openshift.io/cert-manager-operator:latest + containerImage: quay.io/rh-ee-mykastur/cm:v1.0.7 createdAt: 2023-03-03T00:00:00 features.operators.openshift.io/cnf: "false" features.operators.openshift.io/cni: "false" @@ -250,7 +250,6 @@ metadata: features.operators.openshift.io/token-auth-aws: "true" features.operators.openshift.io/token-auth-azure: "true" features.operators.openshift.io/token-auth-gcp: "true" - olm.skipRange: '>=1.17.0 <1.18.0' operator.openshift.io/uninstall-message: The cert-manager Operator for Red Hat OpenShift will be removed from cert-manager-operator namespace. If your Operator configured any off-cluster resources, these will continue to run and require @@ -271,7 +270,7 @@ metadata: operatorframework.io/arch.ppc64le: supported operatorframework.io/arch.s390x: supported operatorframework.io/os.linux: supported - name: cert-manager-operator.v1.18.0 + name: cert-manager-operator.v1.0.7 namespace: cert-manager-operator spec: apiservicedefinitions: {} @@ -656,11 +655,11 @@ spec: - name: OPERATOR_NAME value: cert-manager-operator - name: RELATED_IMAGE_CERT_MANAGER_WEBHOOK - value: quay.io/jetstack/cert-manager-webhook:latest + value: quay.io/jetstack/cert-manager-webhook:v1.18.2 - name: RELATED_IMAGE_CERT_MANAGER_CA_INJECTOR - value: quay.io/jetstack/cert-manager-cainjector:latest + value: quay.io/jetstack/cert-manager-cainjector:v1.18.2 - name: RELATED_IMAGE_CERT_MANAGER_CONTROLLER - value: quay.io/jetstack/cert-manager-controller:latest + value: quay.io/jetstack/cert-manager-controller:v1.18.2 - name: RELATED_IMAGE_CERT_MANAGER_ACMESOLVER value: quay.io/jetstack/cert-manager-acmesolver:v1.18.2 - name: RELATED_IMAGE_CERT_MANAGER_ISTIOCSR @@ -676,7 +675,7 @@ spec: - name: TRUSTED_CA_CONFIGMAP_NAME - name: CLOUD_CREDENTIALS_SECRET_NAME - name: UNSUPPORTED_ADDON_FEATURES - image: openshift.io/cert-manager-operator:latest + image: quay.io/rh-ee-mykastur/cm:v1.0.7 imagePullPolicy: IfNotPresent name: cert-manager-operator ports: @@ -771,15 +770,14 @@ spec: provider: name: Red Hat relatedImages: - - image: quay.io/jetstack/cert-manager-webhook:latest + - image: quay.io/jetstack/cert-manager-webhook:v1.18.2 name: cert-manager-webhook - - image: quay.io/jetstack/cert-manager-cainjector:latest + - image: quay.io/jetstack/cert-manager-cainjector:v1.18.2 name: cert-manager-ca-injector - - image: quay.io/jetstack/cert-manager-controller:latest + - image: quay.io/jetstack/cert-manager-controller:v1.18.2 name: cert-manager-controller - image: quay.io/jetstack/cert-manager-acmesolver:v1.18.2 name: cert-manager-acmesolver - image: quay.io/jetstack/cert-manager-istio-csr:v0.14.2 name: cert-manager-istiocsr - replaces: cert-manager-operator.v1.17.0 - version: 1.18.0 + version: 1.0.7 diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index ab5ff95a..16a9adaa 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -4,4 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: openshift.io/cert-manager-operator + newName: quay.io/rh-ee-mykastur/cm + newTag: v1.0.7 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 2caa9d29..11f084d9 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -75,11 +75,11 @@ spec: - name: OPERATOR_NAME value: cert-manager-operator - name: RELATED_IMAGE_CERT_MANAGER_WEBHOOK - value: quay.io/jetstack/cert-manager-webhook:latest + value: quay.io/jetstack/cert-manager-webhook:v1.18.2 - name: RELATED_IMAGE_CERT_MANAGER_CA_INJECTOR - value: quay.io/jetstack/cert-manager-cainjector:latest + value: quay.io/jetstack/cert-manager-cainjector:v1.18.2 - name: RELATED_IMAGE_CERT_MANAGER_CONTROLLER - value: quay.io/jetstack/cert-manager-controller:latest + value: quay.io/jetstack/cert-manager-controller:v1.18.2 - name: RELATED_IMAGE_CERT_MANAGER_ACMESOLVER value: quay.io/jetstack/cert-manager-acmesolver:v1.18.2 - name: RELATED_IMAGE_CERT_MANAGER_ISTIOCSR diff --git a/config/manifests/bases/cert-manager-operator.clusterserviceversion.yaml b/config/manifests/bases/cert-manager-operator.clusterserviceversion.yaml index 5b705214..07d288d6 100644 --- a/config/manifests/bases/cert-manager-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/cert-manager-operator.clusterserviceversion.yaml @@ -18,7 +18,6 @@ metadata: features.operators.openshift.io/token-auth-aws: "true" features.operators.openshift.io/token-auth-azure: "true" features.operators.openshift.io/token-auth-gcp: "true" - olm.skipRange: '>=1.17.0 <1.18.0' operator.openshift.io/uninstall-message: The cert-manager Operator for Red Hat OpenShift will be removed from cert-manager-operator namespace. If your Operator configured any off-cluster resources, these will continue to run and require @@ -104,5 +103,4 @@ spec: minKubeVersion: 1.27.0 provider: name: Red Hat - replaces: cert-manager-operator.v1.17.0 version: 1.18.0 diff --git a/pkg/controller/deployment/deployment_helper_test.go b/pkg/controller/deployment/deployment_helper_test.go index 4d090e91..517b503b 100644 --- a/pkg/controller/deployment/deployment_helper_test.go +++ b/pkg/controller/deployment/deployment_helper_test.go @@ -1006,7 +1006,7 @@ func TestGetOverrideReplicasFor(t *testing.T) { }, Spec: v1alpha1.CertManagerSpec{ ControllerConfig: &v1alpha1.DeploymentConfig{ - OverrideArgs: []string{"--v=3"}, + OverrideArgs: []string{"--v=3"}, OverrideReplicas: nil, }, }, @@ -1022,7 +1022,7 @@ func TestGetOverrideReplicasFor(t *testing.T) { }, Spec: v1alpha1.CertManagerSpec{ WebhookConfig: &v1alpha1.DeploymentConfig{ - OverrideEnv: []corev1.EnvVar{{Name: "TEST", Value: "value"}}, + OverrideEnv: []corev1.EnvVar{{Name: "TEST", Value: "value"}}, OverrideReplicas: nil, }, }, @@ -1038,7 +1038,7 @@ func TestGetOverrideReplicasFor(t *testing.T) { }, Spec: v1alpha1.CertManagerSpec{ CAInjectorConfig: &v1alpha1.DeploymentConfig{ - OverrideLabels: map[string]string{"test": "label"}, + OverrideLabels: map[string]string{"test": "label"}, OverrideReplicas: nil, }, }, diff --git a/pkg/controller/istiocsr/client.go b/pkg/controller/istiocsr/client.go index c22af42f..e1da7336 100644 --- a/pkg/controller/istiocsr/client.go +++ b/pkg/controller/istiocsr/client.go @@ -39,16 +39,16 @@ func NewClient(m manager.Manager) (ctrlClient, error) { if err != nil { return nil, fmt.Errorf("failed to build custom client: %w", err) } - + // Log that we're using a CUSTOM client (not manager's client) customClientPtr := fmt.Sprintf("%p", c) - klog.Infof("[CACHE-TRACKING] NewClient: Using CUSTOM client (ptr=%s) with SEPARATE custom cache", - customClientPtr) + klog.Infof("[CACHE-TRACKING] NewClient: Using CUSTOM client (ptr=%s) with SEPARATE custom cache (ptr=%s)", + customClientPtr, customCachePtr) klog.Warningf("[CACHE-TRACKING] NewClient: WARNING - Custom client uses DIFFERENT cache than manager! Race condition possible!") - + return &ctrlClientImpl{ - Client: c, - cacheID: "CUSTOM-CACHE", // Will be updated in BuildCustomClient + Client: c, + cacheID: customCachePtr, // Use the actual custom cache pointer }, nil } @@ -57,11 +57,26 @@ func (c *ctrlClientImpl) Get( ) error { klog.V(4).Infof("[CACHE-TRACKING] Get operation: key=%s, type=%T, cacheID=%s", key.String(), obj, c.cacheID) + + // Before Get - cache state unknown + klog.Infof("[CACHE-TRACKING] CACHE-READ: Attempting to read from Cache #2 (custom cache), key=%s, type=%T, cacheID=%s", + key.String(), obj, c.cacheID) + err := c.Client.Get(ctx, key, obj) + if err != nil { klog.V(4).Infof("[CACHE-TRACKING] Get result: key=%s, error=%v, cacheID=%s", key.String(), err, c.cacheID) + if errors.IsNotFound(err) { + klog.Warningf("[CACHE-TRACKING] CACHE-READ: Object NOT FOUND in Cache #2 (custom cache), key=%s, cacheID=%s, reason=Cache #2 may not have synced yet after Cache #1 detected the watch event", + key.String(), c.cacheID) + } + } else { + // Success - object found + klog.Infof("[CACHE-TRACKING] CACHE-READ: Successfully read from Cache #2 (custom cache), key=%s, type=%T, cacheID=%s, note=Cache #2 was synced", + key.String(), obj, c.cacheID) } + return err } diff --git a/pkg/controller/istiocsr/controller.go b/pkg/controller/istiocsr/controller.go index 3a314488..9877c623 100644 --- a/pkg/controller/istiocsr/controller.go +++ b/pkg/controller/istiocsr/controller.go @@ -40,6 +40,9 @@ var ( // requestEnqueueLabelValue is the label value used for filtering reconcile // events to include only the resources created by the controller. requestEnqueueLabelValue = "cert-manager-istio-csr" + + // customCachePtr stores the custom cache pointer for logging + customCachePtr string ) // Reconciler reconciles a IstioCSR object @@ -119,19 +122,30 @@ func BuildCustomClient(mgr ctrl.Manager) (client.Client, error) { if err != nil { return nil, err } - + // Log the SEPARATE custom cache that was created - customCachePtr := fmt.Sprintf("%p", customCache) + customCachePtrLocal := fmt.Sprintf("%p", customCache) + customCachePtr = customCachePtrLocal // Save globally for client.go to use ctrl.Log.WithName("build-custom-client").Info("[CACHE-TRACKING] BuildCustomClient: Created CUSTOM cache (Cache #2)", - "customCachePtr", customCachePtr, + "customCachePtr", customCachePtrLocal, "labelSelector", managedResourceLabelReqSelector.String()) ctrl.Log.WithName("build-custom-client").Info("[CACHE-TRACKING] BuildCustomClient: WARNING - DUAL CACHE detected!", "managerCachePtr", managerCachePtr, - "customCachePtr", customCachePtr, + "customCachePtr", customCachePtrLocal, "racePossible", "YES - watches use manager cache, reconciler uses custom cache") - if _, err = customCache.GetInformer(context.Background(), &v1alpha1.IstioCSR{}); err != nil { + ctrl.Log.WithName("build-custom-client").Info("[CACHE-TRACKING] BuildCustomClient: Registering informers for Cache #2", + "customCachePtr", customCachePtrLocal, + "note", "These informers will populate Cache #2's stores - watch for storePtr values in CACHE-UPDATE logs") + + istiocsrInformer, err := customCache.GetInformer(context.Background(), &v1alpha1.IstioCSR{}) + if err != nil { return nil, err } + // Log informer creation for Cache #2 + ctrl.Log.WithName("build-custom-client").Info("[CACHE-TRACKING] BuildCustomClient: Created IstioCSR informer for Cache #2", + "customCachePtr", customCachePtrLocal, + "informerPtr", fmt.Sprintf("%p", istiocsrInformer), + "note", "This informer will populate Cache #2 with IstioCSR objects") if _, err = customCache.GetInformer(context.Background(), &certmanagerv1.Certificate{}); err != nil { return nil, err } @@ -173,7 +187,7 @@ func BuildCustomClient(mgr ctrl.Manager) (client.Client, error) { if err != nil { return nil, err } - + ctrl.Log.WithName("build-custom-client").Info("[CACHE-TRACKING] BuildCustomClient: Registered custom cache as runnable with manager") customClient, err := client.New(mgr.GetConfig(), client.Options{ @@ -187,7 +201,7 @@ func BuildCustomClient(mgr ctrl.Manager) (client.Client, error) { if err != nil { return nil, err } - + customClientPtr := fmt.Sprintf("%p", customClient) ctrl.Log.WithName("build-custom-client").Info("[CACHE-TRACKING] BuildCustomClient: Created custom client backed by custom cache", "customClientPtr", customClientPtr, @@ -204,11 +218,20 @@ func BuildCustomClient(mgr ctrl.Manager) (client.Client, error) { func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { // Get cache pointer for logging in watch events managerCachePtr := fmt.Sprintf("%p", mgr.GetCache()) - + mapFunc := func(ctx context.Context, obj client.Object) []reconcile.Request { r.log.V(4).Info("received reconcile event", "object", fmt.Sprintf("%T", obj), "name", obj.GetName(), "namespace", obj.GetNamespace()) // Log watch event with manager cache information + r.log.Info("[CACHE-TRACKING] CACHE-UPDATE: Cache #1 (manager) detected object change from API watch", + "objectType", fmt.Sprintf("%T", obj), + "objectName", obj.GetName(), + "objectNamespace", obj.GetNamespace(), + "resourceVersion", obj.GetResourceVersion(), + "cacheID", managerCachePtr, + "cache", "Cache #1 (manager)", + "note", "Cache #1 updated BEFORE triggering reconcile") + r.log.Info("[CACHE-TRACKING] Watch Event: Object change detected", "objectType", fmt.Sprintf("%T", obj), "objectName", obj.GetName(), @@ -251,7 +274,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { Namespace: namespace, }, } - + r.log.Info("[CACHE-TRACKING] Watch Event: Enqueuing reconcile request", "objectType", fmt.Sprintf("%T", obj), "objectName", obj.GetName(), @@ -259,7 +282,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { "cacheID", managerCachePtr, "action", "enqueue-reconcile", "note", "Watch from manager Cache #1, but reconciler will read from custom Cache #2!") - + return []reconcile.Request{reconcileReq} } } @@ -301,13 +324,13 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { Watches(&corev1.ServiceAccount{}, handler.EnqueueRequestsFromMapFunc(mapFunc), controllerManagedResourcePredicates). WatchesMetadata(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(mapFunc), controllerWatchResourcePredicates). Complete(r) - + if err == nil { r.log.Info("[CACHE-TRACKING] SetupWithManager: Watches registered with manager Cache #1", "cacheID", managerCachePtr, "warning", "Reconciler will read from custom Cache #2 - DIFFERENT cache!") } - + return err } @@ -319,23 +342,27 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // Fetch the istiocsr.openshift.operator.io CR istiocsr := &v1alpha1.IstioCSR{} - r.log.Info("[CACHE-TRACKING] Reconcile: About to call r.Get() to fetch IstioCSR from CUSTOM cache (Cache #2)", "request", req) + r.log.Info("[CACHE-TRACKING] Reconcile: About to call r.Get() to fetch IstioCSR from CUSTOM cache (Cache #2)", + "request", req, + "note", "This will read from Cache #2, which may lag behind Cache #1 that triggered this reconcile") + r.log.Info("[CACHE-TRACKING] TIMING: Watch triggered by Cache #1 → Now reading from Cache #2", + "timingNote", "If Cache #2 not synced yet, race condition will manifest") if err := r.Get(ctx, req.NamespacedName, istiocsr); err != nil { if errors.IsNotFound(err) { // NotFound errors, since they can't be fixed by an immediate // requeue (have to wait for a new notification), and can be processed // on deleted requests. r.log.V(1).Info("istiocsr.openshift.operator.io object not found, skipping reconciliation", "request", req) - r.log.Info("[CACHE-TRACKING] Reconcile: IstioCSR NOT FOUND in custom cache (Cache #2) - RACE CONDITION!", + r.log.Info("[CACHE-TRACKING] Reconcile: IstioCSR NOT FOUND in custom cache (Cache #2) - RACE CONDITION!", "request", req, "reason", "Watch triggered from Cache #1 but reconciler reads from Cache #2 which may not be synced yet") return ctrl.Result{}, nil } return ctrl.Result{}, fmt.Errorf("failed to fetch istiocsr.openshift.operator.io %q during reconciliation: %w", req.NamespacedName, err) } - r.log.Info("[CACHE-TRACKING] Reconcile: Successfully fetched IstioCSR from custom cache (Cache #2)", - "request", req, - "generation", istiocsr.Generation, + r.log.Info("[CACHE-TRACKING] Reconcile: Successfully fetched IstioCSR from custom cache (Cache #2)", + "request", req, + "generation", istiocsr.Generation, "resourceVersion", istiocsr.ResourceVersion, "note", "Cache #2 was synced in time - no race this time") diff --git a/vendor/k8s.io/client-go/tools/cache/controller.go b/vendor/k8s.io/client-go/tools/cache/controller.go index e523a665..e7fc081a 100644 --- a/vendor/k8s.io/client-go/tools/cache/controller.go +++ b/vendor/k8s.io/client-go/tools/cache/controller.go @@ -21,9 +21,11 @@ import ( "sync" "time" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" "k8s.io/utils/clock" ) @@ -555,17 +557,33 @@ func processDeltas( if err := clientState.Update(obj); err != nil { return err } + // Log cache update + if accessor, err := meta.Accessor(obj); err == nil { + klog.V(2).Infof("[CACHE-TRACKING] CACHE-UPDATE: Object UPDATED in cache, type=%T, name=%s, namespace=%s, resourceVersion=%s, cacheStorePtr=%p", + obj, accessor.GetName(), accessor.GetNamespace(), accessor.GetResourceVersion(), clientState) + } handler.OnUpdate(old, obj) } else { if err := clientState.Add(obj); err != nil { return err } + // Log cache add + if accessor, err := meta.Accessor(obj); err == nil { + // Log with store pointer - each resource type has its own store + klog.Infof("[CACHE-TRACKING] CACHE-UPDATE: Object ADDED to cache store, type=%T, name=%s, namespace=%s, resourceVersion=%s, storePtr=%p", + obj, accessor.GetName(), accessor.GetNamespace(), accessor.GetResourceVersion(), clientState) + } handler.OnAdd(obj, isInInitialList) } case Deleted: if err := clientState.Delete(obj); err != nil { return err } + // Log cache delete + if accessor, err := meta.Accessor(obj); err == nil { + klog.V(2).Infof("[CACHE-TRACKING] CACHE-UPDATE: Object DELETED from cache, type=%T, name=%s, namespace=%s, cacheStorePtr=%p", + obj, accessor.GetName(), accessor.GetNamespace(), clientState) + } handler.OnDelete(obj) } }