---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  annotations:
    include.release.openshift.io/self-managed-high-availability: "true"
    include.release.openshift.io/single-node-developer: "true"
  creationTimestamp: "2025-10-27T11:18:35Z"
  generation: 1
  managedFields:
  - apiVersion: monitoring.coreos.com/v1
    fieldsType: FieldsV1
    fieldsV1:
      f:metadata:
        f:annotations:
          .: {}
          f:include.release.openshift.io/self-managed-high-availability: {}
          f:include.release.openshift.io/single-node-developer: {}
        f:ownerReferences:
          .: {}
          k:{"uid":"d36bdf59-d268-4238-afb0-5e0b21cca1a4"}: {}
      f:spec:
        .: {}
        f:groups:
          .: {}
          k:{"name":"etcd"}:
            .: {}
            f:name: {}
            f:rules: {}
          k:{"name":"openshift-etcd.rules"}:
            .: {}
            f:name: {}
            f:rules: {}
    manager: cluster-version-operator
    operation: Update
    time: "2025-10-27T11:18:35Z"
  name: etcd-prometheus-rules
  namespace: openshift-etcd-operator
  ownerReferences:
  - apiVersion: config.openshift.io/v1
    controller: true
    kind: ClusterVersion
    name: version
    uid: d36bdf59-d268-4238-afb0-5e0b21cca1a4
  resourceVersion: "1704"
  uid: 64cc26ed-eb75-4ba5-9fed-99eaf901f947
spec:
  groups:
  - name: etcd
    rules:
    - alert: etcdNoLeader
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": member {{ $labels.instance
          }} has no leader.'
        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdNoLeader.md
        summary: etcd cluster has no leader.
      expr: |
        etcd_server_has_leader{job=~".*etcd.*"} == 0
      for: 1m
      labels:
        severity: critical
    - alert: etcdMemberCommunicationSlow
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": member communication with
          {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance
          }}.'
        summary: etcd cluster member communication is slow.
      expr: |
        histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.15
      for: 10m
      labels:
        severity: warning
    - alert: etcdHighNumberOfFailedProposals
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }} proposal failures
          within the last 30 minutes on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster has high number of proposal failures.
      expr: |
        rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
      for: 15m
      labels:
        severity: warning
    - alert: etcdHighFsyncDurations
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations
          are {{ $value }}s on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster 99th percentile fsync durations are too high.
      expr: |
        histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.5
      for: 10m
      labels:
        severity: warning
    - alert: etcdHighFsyncDurations
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations
          are {{ $value }}s on etcd instance {{ $labels.instance }}.'
        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdHighFsyncDurations.md
        summary: etcd cluster 99th percentile fsync durations are too high.
      expr: |
        histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 1
      for: 10m
      labels:
        severity: critical
    - alert: etcdExcessiveDatabaseGrowth
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": Predicting running out of
          disk space in the next four hours, based on write observations within the
          past four hours on etcd instance {{ $labels.instance }}, please check as
          it might be disruptive.'
        summary: etcd cluster database growing very fast.
      expr: |
        predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"}
      for: 10m
      labels:
        severity: warning
    - alert: etcdDatabaseHighFragmentationRatio
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": database size in use on instance
          {{ $labels.instance }} is {{ $value | humanizePercentage }} of the actual
          allocated disk space, please run defragmentation (e.g. etcdctl defrag) to
          retrieve the unused fragmented disk space.'
        runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation
        summary: etcd database size in use is less than 50% of the actual allocated
          storage.
      expr: |
        (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600
      for: 10m
      labels:
        severity: warning
  - name: openshift-etcd.rules
    rules:
    - alert: etcdDatabaseQuotaLowSpace
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": database size is 65% of the
          defined quota on etcd instance {{ $labels.instance }}, please defrag or
          increase the quota as the writes to etcd will be disabled when it is full.'
        summary: etcd cluster database is using >= 65% of the defined quota.
      expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])
        / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100
        > 65
      for: 10m
      labels:
        severity: info
    - alert: etcdDatabaseQuotaLowSpace
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": database size is 75% of the
          defined quota on etcd instance {{ $labels.instance }}, please defrag or
          increase the quota as the writes to etcd will be disabled when it is full.'
        summary: etcd cluster database is using >= 75% of the defined quota.
      expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])
        / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100
        > 75
      for: 10m
      labels:
        severity: warning
    - alert: etcdDatabaseQuotaLowSpace
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": database size is 85% of the
          defined quota on etcd instance {{ $labels.instance }}, please defrag or
          increase the quota as the writes to etcd will be disabled when it is full.'
        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdDatabaseQuotaLowSpace.md
        summary: etcd cluster database is running full.
      expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])
        / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100
        > 85
      for: 10m
      labels:
        severity: critical
    - alert: etcdGRPCRequestsSlow
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC requests
          is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method
          }} method.'
        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdGRPCRequestsSlow.md
        summary: etcd grpc requests are slow
      expr: |
        histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd", grpc_method!="Defragment", grpc_type="unary"}[10m])) without(grpc_type))
        > on () group_left (type)
        bottomk(1,
          1.5 * group by (type) (cluster_infrastructure_provider{type="Azure"})
          or
          1 * group by (type) (cluster_infrastructure_provider))
      for: 30m
      labels:
        severity: critical
    - alert: etcdHighCommitDurations
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations
          {{ $value }}s on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster 99th percentile commit durations are too high.
      expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.5
      for: 10m
      labels:
        severity: warning
    - alert: etcdHighNumberOfFailedGRPCRequests
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests
          for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance
          }}.'
        summary: etcd cluster has high number of failed grpc requests.
      expr: |
        (sum(rate(grpc_server_handled_total{job="etcd", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
          /
        (sum(rate(grpc_server_handled_total{job="etcd"}[5m])) without (grpc_type, grpc_code)
          > 2 and on ()(sum(cluster_infrastructure_provider{type!~"ipi|BareMetal"} == bool 1)))) * 100 > 10
      for: 10m
      labels:
        severity: warning
    - alert: etcdHighNumberOfFailedGRPCRequests
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests
          for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance
          }}.'
        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdHighNumberOfFailedGRPCRequests.md
        summary: etcd cluster has high number of failed grpc requests.
      expr: |
        (sum(rate(grpc_server_handled_total{job="etcd", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
          /
        (sum(rate(grpc_server_handled_total{job="etcd"}[5m])) without (grpc_type, grpc_code)
          > 2 and on ()(sum(cluster_infrastructure_provider{type!~"ipi|BareMetal"} == bool 1)))) * 100 > 50
      for: 10m
      labels:
        severity: critical
    - alert: etcdHighNumberOfLeaderChanges
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }} average leader
          changes within the last 10 minutes. Frequent elections may be a sign of
          insufficient resources, high network latency, or disruptions by other components
          and should be investigated.'
        summary: etcd cluster has high number of leader changes.
      expr: |
        avg(changes(etcd_server_is_leader[10m])) > 5
      for: 5m
      labels:
        severity: warning
    - alert: etcdInsufficientMembers
      annotations:
        description: etcd is reporting fewer instances are available than are needed
          ({{ $value }}). When etcd does not have a majority of instances available
          the Kubernetes and OpenShift APIs will reject read and write requests and
          operations that preserve the health of workloads cannot be performed. This
          can occur when multiple control plane nodes are powered off or are unable
          to connect to each other via the network. Check that all control plane nodes
          are powered on and that network connections between each machine are functional.
        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdInsufficientMembers.md
        summary: etcd is reporting that a majority of instances are unavailable.
      expr: sum(up{job="etcd"} == bool 1 and etcd_server_has_leader{job="etcd"} ==
        bool 1) without (instance,pod) < ((count(up{job="etcd"}) without (instance,pod)
        + 1) / 2)
      for: 3m
      labels:
        severity: critical
    - alert: etcdMembersDown
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": members are down ({{ $value
          }}).'
        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdMembersDown.md
        summary: etcd cluster members are down.
      expr: |
        max without (endpoint) (
            sum without (instance) (up{job=~".*etcd.*"} == bool 0)
        or
          count without (To) (
            sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01
          )
        )
        > 0
      for: 20m
      labels:
        severity: critical
    - alert: etcdSignerCAExpirationWarning
      annotations:
        description: etcd is reporting the signer ca "{{ $labels.name }}" to have
          less than two years (({{ printf "%.f" $value }} days) of validity left.
        summary: etcd signer ca is about to expire
      expr: avg(openshift_etcd_operator_signer_expiration_days) by (name) < 730
      for: 1h
      labels:
        severity: warning
    - alert: etcdSignerCAExpirationCritical
      annotations:
        description: etcd is reporting the signer ca "{{ $labels.name }}" to have
          less than year  (({{ printf "%.f" $value }} days) of validity left.
        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdSignerCAExpirationCritical.md
        summary: etcd has critical signer ca expiration
      expr: avg(openshift_etcd_operator_signer_expiration_days) by (name) < 365
      for: 1h
      labels:
        severity: critical
