launch openshift/monitoring-plugin#53

$ token=`oc create token prometheus-k8s -n openshift-monitoring`
$ oc -n openshift-monitoring exec -c prometheus prometheus-k8s-0 -- curl -k -H "Authorization: Bearer $token" 'https://thanos-querier.openshift-monitoring.svc:9091/api/v1/query?' --data-urlencode 'query=ALERTS{alertname="Watchdog"}' | jq
{
  "status": "success",
  "data": {
    "resultType": "vector",
    "result": [
      {
        "metric": {
          "__name__": "ALERTS",
          "alertname": "Watchdog",
          "alertstate": "firing",
          "namespace": "openshift-monitoring",
          "prometheus": "openshift-monitoring/k8s",
          "severity": "none"
        },
        "value": [
          1693536363.814,
          "1"
        ]
      }
    ]
  }
}

$ oc -n openshift-monitoring exec -c prometheus prometheus-k8s-0 -- cat /etc/prometheus/config_out/prometheus.env.yaml | head -n 7
global:
  evaluation_interval: 30s
  scrape_interval: 30s
  external_labels:
    prometheus: openshift-monitoring/k8s
    prometheus_replica: prometheus-k8s-0
rule_files:  

namespace=openshift-monitoring
prometheus=openshift-monitoring/k8s
severity=none
alertname=Watchdog 

namespace=openshift-monitoring
severity=none
alertname=Watchdog 

$ oc -n openshift-monitoring exec -c prometheus prometheus-k8s-0 -- curl -k -H "Authorization: Bearer $token" 'https://thanos-querier.openshift-monitoring.svc:9091/api/v1/rules' | jq '.data.groups[].rules[] | select(.name=="Watchdog")'
{
  "state": "firing",
  "name": "Watchdog",
  "query": "vector(1)",
  "duration": 0,
  "labels": {
    "namespace": "openshift-monitoring",
    "prometheus": "openshift-monitoring/k8s",
    "severity": "none"
  },
  "annotations": {
    "description": "This is an alert meant to ensure that the entire alerting pipeline is functional.\nThis alert is always firing, therefore it should always be firing in Alertmanager\nand always fire against a receiver. There are integrations with various notification\nmechanisms that send a notification when this alert is not firing. For example the\n\"DeadMansSnitch\" integration in PagerDuty.\n",
    "summary": "An alert that should always be firing to certify that Alertmanager is working properly."
  },
  "alerts": [
    {
      "labels": {
        "alertname": "Watchdog",
        "namespace": "openshift-monitoring",
        "severity": "none"
      },
      "annotations": {
        "description": "This is an alert meant to ensure that the entire alerting pipeline is functional.\nThis alert is always firing, therefore it should always be firing in Alertmanager\nand always fire against a receiver. There are integrations with various notification\nmechanisms that send a notification when this alert is not firing. For example the\n\"DeadMansSnitch\" integration in PagerDuty.\n",
        "summary": "An alert that should always be firing to certify that Alertmanager is working properly."
      },
      "state": "firing",
      "activeAt": "2023-09-01T01:05:18.380033665Z",
      "value": "1e+00",
      "partialResponseStrategy": "WARN"
    }
  ],
  "health": "ok",
  "evaluationTime": 0.000553527,
  "lastEvaluation": "2023-09-01T03:04:18.381381325Z",
  "type": "alerting"
}