Uploaded image for project: 'Distributed Tracing'
  1. Distributed Tracing
  2. TRACING-3568

Investigate tempo reconcile error messages and certificate log spam

XMLWordPrintable

    • Icon: Bug Bug
    • Resolution: Done-Errata
    • Icon: Undefined Undefined
    • rhosdt-3.0
    • None
    • Tempo
    • None
    • 1
    • False
    • None
    • False
    • Tracing Sprint # 243, Tracing Sprint # 244

      Starting with the example CR from the README:

      apiVersion: tempo.grafana.com/v1alpha1
      kind: TempoStack
      metadata:
        name: simplest
      spec:
        storage:
          secret:
            name: minio-test
            type: s3
        storageSize: 1Gi
        resources:
          total:
            limits:
              memory: 2Gi
              cpu: 2000m
        template:
          queryFrontend:
            jaegerQuery:
              enabled: true
      

      If I e.g. increase only the CPU limit, I see the following logs:

      {"level":"info","ts":"2023-09-13T12:39:34.375236946Z","logger":"certrotation-reconcile","msg":"Checking if TempoStack certificates expired","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"de402bec-bebd-48ac-aa5c-b0a340d58799","tempo":{"name":"simplest","namespace":"default"},"name":"default/simplest","interval":"12h0m0s"}
      {"level":"info","ts":"2023-09-13T12:39:34.37562641Z","logger":"certrotation-reconcile","msg":"Skipping cert rotation, all TempoStack certificates still valid","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"de402bec-bebd-48ac-aa5c-b0a340d58799","tempo":{"name":"simplest","namespace":"default"},"name":"default/simplest"}
      {"level":"info","ts":"2023-09-13T12:39:34.375676783Z","logger":"tempostack-reconcile","msg":"certificate manifests built","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"e75ecdf5-b60d-40dc-8d33-95fbc164cd73","tempo":{"name":"simplest","namespace":"default"},"tempostacks":"default/simplest","event":"createOrRotateCerts","count":8}
      {"level":"info","ts":"2023-09-13T12:39:34.509259308Z","logger":"certrotation-reconcile","msg":"Checking if TempoStack certificates expired","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"cad4f8e7-8a76-4465-8982-76163cce2d45","tempo":{"name":"simplest","namespace":"default"},"name":"default/simplest","interval":"12h0m0s"}
      {"level":"info","ts":"2023-09-13T12:39:34.509647717Z","logger":"tempostack-reconcile","msg":"certificate manifests built","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"1c63f09b-c2f2-42c3-810b-e542fa38b73d","tempo":{"name":"simplest","namespace":"default"},"tempostacks":"default/simplest","event":"createOrRotateCerts","count":8}
      {"level":"info","ts":"2023-09-13T12:39:34.509890166Z","logger":"certrotation-reconcile","msg":"Skipping cert rotation, all TempoStack certificates still valid","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"cad4f8e7-8a76-4465-8982-76163cce2d45","tempo":{"name":"simplest","namespace":"default"},"name":"default/simplest"}
      {"level":"error","ts":"2023-09-13T12:39:34.587606545Z","logger":"tempostack-reconcile","msg":"failed to configure resource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"1c63f09b-c2f2-42c3-810b-e542fa38b73d","tempo":{"name":"simplest","namespace":"default"},"object_name":"tempo-simplest-querier","object_kind":"&TypeMeta{Kind:,APIVersion:apps/v1,}","error":"Operation cannot be fulfilled on deployments.apps \"tempo-simplest-querier\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/grafana/tempo-operator/controllers/tempo.(*TempoStackReconciler).createOrUpdate\n\t/workspace/controllers/tempo/tempostack_create_or_update.go:169\ngithub.com/grafana/tempo-operator/controllers/tempo.(*TempoStackReconciler).Reconcile\n\t/workspace/controllers/tempo/tempostack_controller.go:115\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.16.2/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.16.2/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.16.2/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.16.2/pkg/internal/controller/controller.go:227"}
      {"level":"error","ts":"2023-09-13T12:39:34.605115755Z","msg":"Reconciler error","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"1c63f09b-c2f2-42c3-810b-e542fa38b73d","error":"failed to create objects for TempoStack default/simplest: Operation cannot be fulfilled on deployments.apps \"tempo-simplest-querier\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.16.2/pkg/internal/controller/controller.go:329\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.16.2/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.16.2/pkg/internal/controller/controller.go:227"}
      {"level":"info","ts":"2023-09-13T12:39:34.605208147Z","logger":"certrotation-reconcile","msg":"Checking if TempoStack certificates expired","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"b15c65ba-a7a6-4198-b009-eb373c94af58","tempo":{"name":"simplest","namespace":"default"},"name":"default/simplest","interval":"12h0m0s"}
      {"level":"info","ts":"2023-09-13T12:39:34.605567402Z","logger":"certrotation-reconcile","msg":"Skipping cert rotation, all TempoStack certificates still valid","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"b15c65ba-a7a6-4198-b009-eb373c94af58","tempo":{"name":"simplest","namespace":"default"},"name":"default/simplest"}
      {"level":"info","ts":"2023-09-13T12:39:34.605574512Z","logger":"tempostack-reconcile","msg":"certificate manifests built","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"33b46c81-7dbe-4f01-b49b-ee5943b98195","tempo":{"name":"simplest","namespace":"default"},"tempostacks":"default/simplest","event":"createOrRotateCerts","count":8}
      {"level":"info","ts":"2023-09-13T12:39:34.653764281Z","logger":"certrotation-reconcile","msg":"Checking if TempoStack certificates expired","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"37875325-1802-4a13-aa61-2ae2a8e4bc7b","tempo":{"name":"simplest","namespace":"default"},"name":"default/simplest","interval":"12h0m0s"}
      {"level":"info","ts":"2023-09-13T12:39:34.654103274Z","logger":"certrotation-reconcile","msg":"Skipping cert rotation, all TempoStack certificates still valid","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"37875325-1802-4a13-aa61-2ae2a8e4bc7b","tempo":{"name":"simplest","namespace":"default"},"name":"default/simplest"}
      {"level":"info","ts":"2023-09-13T12:39:34.654102576Z","logger":"tempostack-reconcile","msg":"certificate manifests built","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"8e69c763-4524-42f7-a8d1-2d239610fced","tempo":{"name":"simplest","namespace":"default"},"tempostacks":"default/simplest","event":"createOrRotateCerts","count":8}
      {"level":"info","ts":"2023-09-13T12:39:34.677580717Z","logger":"tempostack-reconcile","msg":"certificate manifests built","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","TempoStack":{"name":"simplest","namespace":"default"},"namespace":"default","name":"simplest","reconcileID":"be34ab58-cb08-4a8f-b950-6d6ef3b132c9","tempo":{"name":"simplest","namespace":"default"},"tempostacks":"default/simplest","event":"createOrRotateCerts","count":8}
      

            agerstma@redhat.com Andreas Gerstmayr
            agerstma@redhat.com Andreas Gerstmayr
            Ishwar Kanse Ishwar Kanse
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

              Created:
              Updated:
              Resolved: