Cluster logging 5.8.20 on OCP 4.15 with fluentd CSV and Pods: ``` [root@rdr-jeev-415log-bastion-0 ~]# oc get csv -n openshift-logging NAME DISPLAY VERSION REPLACES PHASE cluster-logging.v5.8.20 Red Hat OpenShift Logging 5.8.20 cluster-logging.v5.8.19 Succeeded elasticsearch-operator.v5.8.20 OpenShift Elasticsearch Operator 5.8.20 elasticsearch-operator.v5.8.19 Succeeded loki-operator.v5.8.20 Loki Operator 5.8.20 loki-operator.v5.8.19 Succeeded [root@rdr-jeev-415log-bastion-0 ~]# oc get pods -n openshift-logging NAME READY STATUS RESTARTS AGE cluster-logging-operator-59cc8b7874-kvbl2 1/1 Running 0 8h collector-5twz8 1/1 Running 0 4h16m collector-6r8ds 1/1 Running 0 4h16m collector-cnd8g 1/1 Running 0 4h16m collector-qtnfd 1/1 Running 0 4h16m collector-r8ks4 1/1 Running 0 4h16m elasticsearch-cdm-mbsawap4-1-5df648489d-dtfn5 2/2 Running 0 4h24m elasticsearch-cdm-mbsawap4-2-7995d8f899-zx4ds 2/2 Running 0 4h24m elasticsearch-cdm-mbsawap4-3-7c79bc54fb-9jj2k 2/2 Running 0 4h24m elasticsearch-im-app-29120715-zmfw8 0/1 Completed 0 3m50s elasticsearch-im-audit-29120715-4xm2g 0/1 Completed 0 3m50s elasticsearch-im-infra-29120715-f4vzk 0/1 Completed 0 3m50s kibana-5966699dcb-ksg64 2/2 Running 0 4h24m [root@rdr-jeev-415log-bastion-0 ~]# ``` Cluster logging: CloudWatch: Application logs: ``` {'timestamp': 1747214182414, 'message': '{"@timestamp":"2025-05-14T08:35:02.689457231Z","group_name":"rdr-jeev-415log-4hmjl.application","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.196/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:c4\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"169.254.169.5/32\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.196/23\\",\\"gateway_ip\\":\\"10.131.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.196\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:c4\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://68193122fcbb1bc1675fcc7c6853ecb1a157565da689faf213290a75001f95f0","container_image":"quay.io/powercloud/acmeair-authservice-java:1.0","container_name":"acmeair-authservice-java","labels":{"name":"acmeair-auth-deployment","pod-template-hash":"6678875bd5"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"134ae64b-1782-47e2-b155-510e40442fd2","pod_ip":"10.131.0.196","pod_name":"acmeair-authservice-6678875bd5-7pmg7","pod_owner":"ReplicaSet/acmeair-authservice-6678875bd5"},"level":"default","log_type":"application","message":"[AUDIT ] CWWKF0011I: The defaultServer server is ready to run a smarter planet. The defaultServer server started in 11.401 seconds.","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"logs":"5.8_415-application"},"sequence":37544},"stream_name":"kubernetes.var.log.pods.acme-air_acmeair-authservice-6678875bd5-7pmg7_134ae64b-1782-47e2-b155-510e40442fd2.acmeair-authservice-java.0.log"}', 'ingestionTime': 1747214192964} {'timestamp': 1747214170682, 'message': '{"@timestamp":"2025-05-14T08:37:04.213184273Z","group_name":"rdr-jeev-415log-4hmjl.application","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.193/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:c1\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"169.254.169.5/32\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.193/23\\",\\"gateway_ip\\":\\"10.131.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.193\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:c1\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://0ba4b84d716bc3fbd2039f4a368f1e4ab38938c861c20d15cfbb5523e319315f","container_image":"ibmcom/icp-mongodb-ppc64le:4.0.12","container_name":"acmeair-booking-db","labels":{"name":"acmeair-booking-db","pod-template-hash":"5ff54549bb"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"78cfe5de-9126-4d6b-8210-6d9abb16a564","pod_ip":"10.131.0.193","pod_name":"acmeair-booking-db-5ff54549bb-xxld2","pod_owner":"ReplicaSet/acmeair-booking-db-5ff54549bb"},"level":"default","log_type":"application","message":"2025-05-14T08:37:04.213+0000 I NETWORK [conn3] received client metadata from 10.131.0.191:51916 conn3: { driver: { name: \\"mongo-java-driver|legacy\\", version: \\"3.10.1\\" }, os: { type: \\"Linux\\", name: \\"Linux\\", architecture: \\"ppc64le\\", version: \\"5.14.0-284.115.1.el9_2.ppc64le\\" }, platform: \\"Java/Eclipse OpenJ9/1.8.0_262-b10\\" }","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"logs":"5.8_415-application"},"sequence":33750},"stream_name":"kubernetes.var.log.pods.acme-air_acmeair-booking-db-5ff54549bb-xxld2_78cfe5de-9126-4d6b-8210-6d9abb16a564.acmeair-booking-db.0.log"}', 'ingestionTime': 1747214178652} {'timestamp': 1747214170683, 'message': '{"@timestamp":"2025-05-14T08:35:15.119180631Z","group_name":"rdr-jeev-415log-4hmjl.application","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.191/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:bf\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"169.254.169.5/32\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.191/23\\",\\"gateway_ip\\":\\"10.131.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.191\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:bf\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://d1068f3316f722e7600fa813438dec20bc7ab401af2598810dc93705de889c63","container_image":"quay.io/powercloud/acmeair-bookingservice-java:1.0","container_name":"acmeair-bookingservice-java","labels":{"name":"acmeair-booking-deployment","pod-template-hash":"686b7cb778"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"fbe48309-07b0-4451-8843-fa9447439f30","pod_ip":"10.131.0.191","pod_name":"acmeair-bookingservice-686b7cb778-2zlkm","pod_owner":"ReplicaSet/acmeair-bookingservice-686b7cb778"},"level":"warn","log_type":"application","message":"[WARNING ] No suitable sender found. Using NoopSender, meaning that data will not be sent anywhere!","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"logs":"5.8_415-application"},"sequence":33787},"stream_name":"kubernetes.var.log.pods.acme-air_acmeair-bookingservice-686b7cb778-2zlkm_fbe48309-07b0-4451-8843-fa9447439f30.acmeair-bookingservice-java.0.log"}', 'ingestionTime': 1747214179813} {'timestamp': 1747214170683, 'message': '{"@timestamp":"2025-05-14T08:35:03.650518748Z","group_name":"rdr-jeev-415log-4hmjl.application","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.198/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:c6\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"169.254.169.5/32\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.198/23\\",\\"gateway_ip\\":\\"10.131.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.198\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:c6\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://df6e4d70e1182523c54d1deca6bbf2a5c6c1560c45ae1ac729e795cdb8fce93b","container_image":"quay.io/powercloud/acmeair-customerservice-java:1.0","container_name":"acmeair-customerservice-java","labels":{"name":"acmeair-customer-deployment","pod-template-hash":"54ffbc7797"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"d012d864-1a65-4751-b079-3071ce335cc8","pod_ip":"10.131.0.198","pod_name":"acmeair-customerservice-54ffbc7797-zfk9f","pod_owner":"ReplicaSet/acmeair-customerservice-54ffbc7797"},"level":"default","log_type":"application","message":"[AUDIT ] CWWKF0011I: The defaultServer server is ready to run a smarter planet. The defaultServer server started in 12.355 seconds.","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"logs":"5.8_415-application"},"sequence":33767},"stream_name":"kubernetes.var.log.pods.acme-air_acmeair-customerservice-54ffbc7797-zfk9f_d012d864-1a65-4751-b079-3071ce335cc8.acmeair-customerservice-java.0.log"}', 'ingestionTime': 1747214179618} ``` Audit logs: ``` {'timestamp': 1747214329303, 'message': '{"@timestamp":"2025-05-14T09:18:42.787720139Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \\"elasticsearch-operator.v-8yVvKvYJWxi8HPN4l2RIOONrojEfQtEzU5DfVO\\" of ClusterRole \\"elasticsearch-operator.v-8yVvKvYJWxi8HPN4l2RIOONrojEfQtEzU5DfVO\\" to ServiceAccount \\"elasticsearch-operator/openshift-operators-redhat\\""},"apiVersion":"audit.k8s.io/v1","auditID":"b6375274-a7b0-46e3-9d25-be5533e738f0","group_name":"rdr-jeev-415log-4hmjl.audit","hostname":"master-0","k8s_audit_level":"Metadata","kind":"Event","level":"Metadata","log_type":"audit","objectRef":{"apiGroup":"rbac.authorization.k8s.io","apiVersion":"v1","name":"elasticsearch-index-management","namespace":"openshift-logging","resource":"rolebindings","resourceVersion":"5632908","uid":"ea5e4e75-dc85-4ce3-9da2-2543f43019e8"},"openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"logs":"5.8_415-audit"},"sequence":264358},"requestReceivedTimestamp":"2025-05-14T09:18:42.775297Z","requestURI":"/apis/rbac.authorization.k8s.io/v1/namespaces/openshift-logging/rolebindings/elasticsearch-index-management","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.20.177.189"],"stage":"ResponseComplete","stageTimestamp":"2025-05-14T09:18:42.779921Z","stream_name":"master-0.k8s-audit.log","user":{"extra":{"authentication.kubernetes.io/pod-name":["elasticsearch-operator-54fdcdfb47-rrv2t"],"authentication.kubernetes.io/pod-uid":["dedfe19e-617a-4fb8-bbe6-90f14d8051fb"]},"groups":["system:serviceaccounts","system:serviceaccounts:openshift-operators-redhat","system:authenticated"],"uid":"77fbbf3c-16c8-4c36-bd7b-a57817c5ea17","username":"system:serviceaccount:openshift-operators-redhat:elasticsearch-operator"},"userAgent":"elasticsearch-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","verb":"update"}', 'ingestionTime': 1747214331133} {'timestamp': 1747214151258, 'message': '{"@timestamp":"2025-05-14T09:15:47.978+00:00","audit.linux":{"record_id":"68161","type":"PROCTITLE"},"group_name":"rdr-jeev-415log-4hmjl.audit","hostname":"master-0","level":"default","log_type":"audit","message":"type=PROCTITLE msg=audit(1747214147.978:68161): proctitle=2F7573722F62696E2F72756E63002D2D73797374656D642D6367726F7570002D2D726F6F743D2F72756E2F72756E6300637265617465002D2D62756E646C65002F72756E2F636F6E7461696E6572732F73746F726167652F6F7665726C61792D636F6E7461696E6572732F313735613361393063656533376335366463623465","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"logs":"5.8_415-audit"},"sequence":2213},"stream_name":"master-0.linux-audit.log"}', 'ingestionTime': 1747214163018} {'timestamp': 1747214331009, 'message': '{"@timestamp":"2025-05-14T09:18:22.214608900Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \\"system:kube-controller-manager\\" of ClusterRole \\"system:kube-controller-manager\\" to User \\"system:kube-controller-manager\\""},"apiVersion":"audit.k8s.io/v1","auditID":"c84be85f-2350-4ea6-bad2-d53dee501c77","group_name":"rdr-jeev-415log-4hmjl.audit","hostname":"master-0","kind":"Event","level":"Metadata","log_type":"audit","objectRef":{"apiGroup":"oauth.openshift.io","apiVersion":"v1","resource":"useroauthaccesstokens"},"openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"logs":"5.8_415-audit"},"sequence":265425},"openshift_audit_level":"Metadata","requestReceivedTimestamp":"2025-05-12T00:28:36.594347Z","requestURI":"/apis/oauth.openshift.io/v1/useroauthaccesstokens?allowWatchBookmarks=true&resourceVersion=2793123&timeout=9m24s&timeoutSeconds=564&watch=true","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.20.177.13","10.130.0.2"],"stage":"ResponseComplete","stageTimestamp":"2025-05-12T00:38:00.599513Z","stream_name":"master-0.openshift-audit.log","user":{"groups":["system:authenticated"],"username":"system:kube-controller-manager"},"userAgent":"kube-controller-manager/v1.28.15+f383677 (linux/ppc64le) kubernetes/4cf5291/kube-controller-manager","verb":"watch"}', 'ingestionTime': 1747214331448} {'timestamp': 1747214329816, 'message': '{"@timestamp":"2025-05-14T09:18:02.549385896Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \\"cluster-version-operator\\" of ClusterRole \\"cluster-admin\\" to ServiceAccount \\"default/openshift-cluster-version\\""},"apiVersion":"audit.k8s.io/v1","auditID":"7affa9ec-7f2d-41a7-a4fb-c1aacf8fe33e","group_name":"rdr-jeev-415log-4hmjl.audit","hostname":"master-1","k8s_audit_level":"Metadata","kind":"Event","level":"Metadata","log_type":"audit","objectRef":{"apiGroup":"apiextensions.k8s.io","apiVersion":"v1","name":"oauths.config.openshift.io","resource":"customresourcedefinitions"},"openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"logs":"5.8_415-audit"},"sequence":187856},"requestReceivedTimestamp":"2025-05-14T08:58:52.011007Z","requestURI":"/apis/apiextensions.k8s.io/v1/customresourcedefinitions/oauths.config.openshift.io","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.20.177.13"],"stage":"ResponseComplete","stageTimestamp":"2025-05-14T08:58:52.015869Z","stream_name":"master-1.k8s-audit.log","user":{"extra":{"authentication.kubernetes.io/pod-name":["cluster-version-operator-5bc4cc45cf-hzzqq"],"authentication.kubernetes.io/pod-uid":["49d39c13-58ed-4f90-bb25-6c2681afb9eb"]},"groups":["system:serviceaccounts","system:serviceaccounts:openshift-cluster-version","system:authenticated"],"uid":"d8721e8e-7f9b-46b5-8a50-6b68106474d9","username":"system:serviceaccount:openshift-cluster-version:default"},"userAgent":"cluster-version-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","verb":"get"}', 'ingestionTime': 1747214330225} ``` Infrastructure logs: ``` {'timestamp': 1747214331720, 'message': '{"@timestamp":"2025-05-03T19:38:04.693003Z","_RUNTIME_SCOPE":"system","_STREAM_ID":"238fa1b3a68f4299bc0fcee34621021f","_SYSTEMD_INVOCATION_ID":"503588e0ea784a74bbc5d1df4294cf4e","group_name":"rdr-jeev-415log-4hmjl.infrastructure","hostname":"master-0","level":"info","log_type":"infrastructure","message":"time=\\"2025-05-03 19:38:04.692952112Z\\" level=info msg=\\"Checking image status: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:546b3635ab9f651032bf142e243809d64391ee7fcdec9c3d0636218c3dd7aa08\\" id=71b8e5e6-ab8b-456d-8854-f614e93b92f9 name=/runtime.v1.ImageService/ImageStatus","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"logs":"5.8_415-infrastructure"},"sequence":266796},"stream_name":"master-0.journal.system","systemd":{"t":{"BOOT_ID":"4586239920664feb9666ba4c8e16d66c","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/bin/crio","COMM":"crio","EXE":"/usr/bin/crio","GID":"0","MACHINE_ID":"391a7bcf45e544029be895a9654ed7ee","PID":"2472","SELINUX_CONTEXT":"system_u:system_r:container_runtime_t:s0","STREAM_ID":"238fa1b3a68f4299bc0fcee34621021f","SYSTEMD_CGROUP":"/system.slice/crio.service","SYSTEMD_INVOCATION_ID":"503588e0ea784a74bbc5d1df4294cf4e","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"crio.service","TRANSPORT":"stdout","UID":"0"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"crio"}},"time":"2025-05-03T19:38:04+00:00"}', 'ingestionTime': 1747214333175} {'timestamp': 1747214263174, 'message': '{"@timestamp":"2025-05-14T09:16:03.308329553Z","group_name":"rdr-jeev-415log-4hmjl.infrastructure","hostname":"master-0","kubernetes":{"annotations":{"openshift.io/required-scc":"privileged","openshift.io/scc":"privileged"},"container_id":"cri-o://343e6d8e35ace99eb8896ec367df2bb6e51a03aa99d084d906251b63ec21b46a","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:28d53ac4d1793b0d449f7e90f813711d25802b0296dcfb57021e8a9a91da89cf","container_name":"node-ca","labels":{"controller-revision-hash":"798f4784dd","name":"node-ca","pod-template-generation":"1"},"namespace_id":"4e15b04f-023c-47f3-8303-363e8ffbe86f","namespace_labels":{"kubernetes_io_metadata_name":"openshift-image-registry","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-image-registry","pod_id":"cbc91190-6438-45e7-b3da-bc07c6508094","pod_ip":"10.20.177.64","pod_name":"node-ca-dfnjr","pod_owner":"DaemonSet/node-ca"},"level":"default","log_type":"infrastructure","message":"image-registry.openshift-image-registry.svc:5000","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"logs":"5.8_415-infrastructure"},"sequence":171174},"stream_name":"master-0.kubernetes.var.log.pods.openshift-image-registry_node-ca-dfnjr_cbc91190-6438-45e7-b3da-bc07c6508094.node-ca.0.log"}', 'ingestionTime': 1747214266900} {'timestamp': 1747214331993, 'message': '{"@timestamp":"2025-05-11T08:43:18.454712224Z","group_name":"rdr-jeev-415log-4hmjl.infrastructure","hostname":"master-0","kubernetes":{"annotations":{"kubectl.kubernetes.io/default-container":"kube-apiserver","kubernetes.io/config.hash":"1210a5ef9511b99f9768e1c6b57080b1","kubernetes.io/config.mirror":"1210a5ef9511b99f9768e1c6b57080b1","kubernetes.io/config.seen":"2025-05-03T05:26:51.726371426Z","kubernetes.io/config.source":"file","target.workload.openshift.io/management":"{\\"effect\\": \\"PreferredDuringScheduling\\"}"},"container_id":"cri-o://10d97fa356f00048ae77a7ff4f0df12db7765eaad334afbb6a7d9518cc6b340d","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fea1716dc544df53dfdad9ba8c8402494195cde8b6eb3b4c95975dc158a12e9a","container_name":"kube-apiserver","labels":{"apiserver":"true","app":"openshift-kube-apiserver","revision":"15"},"namespace_id":"3cb4e6a8-7c5f-4db1-8e7e-607248006ad5","namespace_labels":{"kubernetes_io_metadata_name":"openshift-kube-apiserver","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-kube-apiserver","pod_id":"99f25828-7172-4ea8-95fc-c0b10763a387","pod_ip":"10.20.177.64","pod_name":"kube-apiserver-master-0","pod_owner":"Node/master-0"},"level":"info","log_type":"infrastructure","message":"I0511 08:43:18.454647 17 controller.go:223] Updating CRD OpenAPI spec because metal3remediations.infrastructure.cluster.x-k8s.io changed","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"logs":"5.8_415-infrastructure"},"sequence":267478},"stream_name":"master-0.kubernetes.var.log.pods.openshift-kube-apiserver_kube-apiserver-master-0_1210a5ef9511b99f9768e1c6b57080b1.kube-apiserver.0.log"}', 'ingestionTime': 1747214334179} {'timestamp': 1747214263174, 'message': '{"@timestamp":"2025-05-02T05:59:58.530046868Z","group_name":"rdr-jeev-415log-4hmjl.infrastructure","hostname":"master-0","kubernetes":{"annotations":{"kubectl.kubernetes.io/default-container":"node-exporter","openshift.io/required-scc":"node-exporter","openshift.io/scc":"node-exporter","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_name":"init-textfile","labels":{"app_kubernetes_io_component":"exporter","app_kubernetes_io_managed-by":"cluster-monitoring-operator","app_kubernetes_io_name":"node-exporter","app_kubernetes_io_part-of":"openshift-monitoring","app_kubernetes_io_version":"1.7.0","controller-revision-hash":"57cb86556","pod-template-generation":"1"},"namespace_id":"29262c15-e202-4ba7-ac6e-3b833726c6fd","namespace_labels":{"kubernetes_io_metadata_name":"openshift-monitoring","name":"openshift-monitoring","network_openshift_io_policy-group":"monitoring","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-monitoring","pod_id":"6534331c-abed-49c2-a7e8-0e73a5a53013","pod_ip":"10.20.177.64","pod_name":"node-exporter-hbdkf","pod_owner":"DaemonSet/node-exporter"},"level":"default","log_type":"infrastructure","message":"/node_exporter/collectors/init/virt.sh: line 47: dmidecode: command not found","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"logs":"5.8_415-infrastructure"},"sequence":171176},"stream_name":"master-0.kubernetes.var.log.pods.openshift-monitoring_node-exporter-hbdkf_6534331c-abed-49c2-a7e8-0e73a5a53013.init-textfile.0.log"}', 'ingestionTime': 1747214268167} ``` Kafka: Application logs: ``` {"@timestamp":"2025-05-14T18:42:58.100707136+00:00","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","docker":{"container_id":"d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"test-logging","pod_name":"centos-logtest-bn5bw","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"deb41613-5199-4569-8d78-7cb401a303f9","pod_ip":"10.131.0.195","host":"worker-0","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"unknown","hostname":"worker-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:42:58.102086+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2240,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-app","logType":"application"}},"viaq_msg_id":"YTAwYTE2NmItNGFhMy00OWQ0LWEzM2MtZDVkOTlmYjYzOGI3","log_type":"application"} {"@timestamp":"2025-05-14T18:42:59.102119840+00:00","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","docker":{"container_id":"d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"test-logging","pod_name":"centos-logtest-bn5bw","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"deb41613-5199-4569-8d78-7cb401a303f9","pod_ip":"10.131.0.195","host":"worker-0","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"unknown","hostname":"worker-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:42:59.104707+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2241,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-app","logType":"application"}},"viaq_msg_id":"ZWQyZTkyYzgtY2U3MS00OWIwLWE1YzktOTE2YTdkY2VkMmRk","log_type":"application"} {"@timestamp":"2025-05-14T18:43:00.103339089+00:00","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","docker":{"container_id":"d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"test-logging","pod_name":"centos-logtest-bn5bw","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"deb41613-5199-4569-8d78-7cb401a303f9","pod_ip":"10.131.0.195","host":"worker-0","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"unknown","hostname":"worker-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:43:00.104742+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2248,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-app","logType":"application"}},"viaq_msg_id":"MjQ4ZGIwODMtMGFiOC00N2FkLWExYjctOTJmMTNhMTI4MThk","log_type":"application"} {"@timestamp":"2025-05-14T18:43:01.104787585+00:00","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","docker":{"container_id":"d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"test-logging","pod_name":"centos-logtest-bn5bw","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"deb41613-5199-4569-8d78-7cb401a303f9","pod_ip":"10.131.0.195","host":"worker-0","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"unknown","hostname":"worker-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:43:01.106795+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2250,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-app","logType":"application"}},"viaq_msg_id":"MmEzM2IzOGUtZjMwNi00YzJlLWFkYmUtZTQ4NjYyOTA0OGE5","log_type":"application"} {"@timestamp":"2025-05-14T18:43:02.106587559+00:00","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","docker":{"container_id":"d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"test-logging","pod_name":"centos-logtest-bn5bw","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"deb41613-5199-4569-8d78-7cb401a303f9","pod_ip":"10.131.0.195","host":"worker-0","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"unknown","hostname":"worker-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:43:02.109148+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2255,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-app","logType":"application"}},"viaq_msg_id":"NTQxY2VhNTItMDMzNC00NGM2LWFmODUtMzU1YzgzZDYzMTU1","log_type":"application"} ``` Audit logs: ``` {"hostname":"worker-0","audit.linux":{"type":"SYSCALL","record_id":"26481"},"message":"type=SYSCALL msg=audit(1747248301.932:26481): arch=c0000015 syscall=361 success=yes exit=14 a0=5 a1=c0001cd958 a2=78 a3=0 items=0 ppid=3458125 pid=3458127 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"runc\" exe=\"/usr/bin/runc\" subj=system_u:system_r:container_runtime_t:s0 key=(null)\u001dARCH=ppc64le SYSCALL=bpf AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\"","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:45:01.969774+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2833,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-audit","logType":"audit"}},"@timestamp":"2025-05-14T18:45:01.931999+00:00","viaq_msg_id":"ZWQxODFiNjktMTMwOS00ZjVhLThjOWYtMmQ5NzBmYTllYWE0","log_type":"audit"} {"hostname":"worker-0","audit.linux":{"type":"PROCTITLE","record_id":"26481"},"message":"type=PROCTITLE msg=audit(1747248301.932:26481): proctitle=2F7573722F62696E2F72756E63002D2D73797374656D642D6367726F7570002D2D726F6F743D2F72756E2F72756E6300637265617465002D2D62756E646C65002F72756E2F636F6E7461696E6572732F73746F726167652F6F7665726C61792D636F6E7461696E6572732F393938633430623165313061333364363230306133","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:45:01.969814+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2834,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-audit","logType":"audit"}},"@timestamp":"2025-05-14T18:45:01.931999+00:00","viaq_msg_id":"YTVkNmRhNGMtNDI3NC00MTdhLTlkMjQtZjYzYzcxOGY5ZmEy","log_type":"audit"} {"hostname":"worker-0","audit.linux":{"type":"BPF","record_id":"26482"},"message":"type=BPF msg=audit(1747248304.782:26482): prog-id=0 op=UNLOAD","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:45:04.791493+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2868,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-audit","logType":"audit"}},"@timestamp":"2025-05-14T18:45:04.782000+00:00","viaq_msg_id":"OWZjOTI0OTktYTI4OS00OTFkLWI3NTYtNzI3MjlkZTVhMmQ3","log_type":"audit"} {"hostname":"worker-0","audit.linux":{"type":"BPF","record_id":"26483"},"message":"type=BPF msg=audit(1747248304.882:26483): prog-id=0 op=UNLOAD","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:45:04.884635+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2871,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-audit","logType":"audit"}},"@timestamp":"2025-05-14T18:45:04.881999+00:00","viaq_msg_id":"OTRlNGE2YzEtZDdhZi00MGQyLWIyZDQtNjUxNjRmZmQxOTFk","log_type":"audit"} {"hostname":"worker-0","audit.linux":{"type":"BPF","record_id":"26484"},"message":"type=BPF msg=audit(1747248305.302:26484): prog-id=0 op=UNLOAD","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:45:05.309564+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2876,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-audit","logType":"audit"}},"@timestamp":"2025-05-14T18:45:05.302000+00:00","viaq_msg_id":"NDUzYmEyNDgtY2Y0ZS00MjVhLWI1YTMtMzRiOGU3ZDEyNWI1","log_type":"audit"} {"hostname":"worker-0","audit.linux":{"type":"BPF","record_id":"26485"},"message":"type=BPF msg=audit(1747248305.412:26485): prog-id=0 op=UNLOAD","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:45:05.414744+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2881,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-audit","logType":"audit"}},"@timestamp":"2025-05-14T18:45:05.411999+00:00","viaq_msg_id":"OTdkMmU1YTEtYmYzOC00OGIwLThhMmYtODJmZTVhZTY4MjBm","log_type":"audit"} {"hostname":"worker-0","audit.linux":{"type":"BPF","record_id":"26486"},"message":"type=BPF msg=audit(1747248305.782:26486): prog-id=0 op=UNLOAD","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:45:05.791781+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2884,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-audit","logType":"audit"}},"@timestamp":"2025-05-14T18:45:05.782000+00:00","viaq_msg_id":"YzdlMDc0MDUtYzE2NC00MGVmLWE5NGItYzA4YWMzNzUwZGJk","log_type":"audit"} {"hostname":"worker-0","audit.linux":{"type":"BPF","record_id":"26487"},"message":"type=BPF msg=audit(1747248305.862:26487): prog-id=0 op=UNLOAD","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:45:05.865419+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2896,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-audit","logType":"audit"}},"@timestamp":"2025-05-14T18:45:05.861999+00:00","viaq_msg_id":"MDQyOWI2NzUtZmNiMi00YmRhLWEwOGYtYjI1NDZhMWQ2N2M1","log_type":"audit"} ``` Infrastructure logs: ``` {"@timestamp":"2025-05-14T18:59:34.999752915+00:00","message":"I0514 18:59:34.999727 1 log.go:194] reconciling (/v1, Kind=Namespace) /openshift-host-network","docker":{"container_id":"106a6ca04bf159cbd7800d42c0d7224da4ae75733679c31ae830b749e4cbe380"},"kubernetes":{"container_name":"network-operator","namespace_name":"openshift-network-operator","pod_name":"network-operator-768fcf7769-xzl99","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bd9a861749b48f743a5e9e6184fbc0b7fcb0011c5759755f3ff85f4ea383b0f","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bd9a861749b48f743a5e9e6184fbc0b7fcb0011c5759755f3ff85f4ea383b0f","pod_id":"c268c47e-c067-44a5-b366-34cd85e7e54f","pod_ip":"10.20.177.159","host":"master-1","labels":{"name":"network-operator","pod-template-hash":"768fcf7769"},"master_url":"https://kubernetes.default.svc","namespace_id":"bdcb252c-d252-4e2a-a7ac-6579f32c386c","namespace_labels":{"kubernetes_io_metadata_name":"openshift-network-operator","name":"openshift-network-operator","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["name=network-operator","pod-template-hash=768fcf7769"]},"level":"info","hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.159","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:59:35.002178+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":87331,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-infra","logType":"infra"}},"viaq_msg_id":"NTM5ZGU1YWYtOTAxZi00ZmY3LTlhOGUtMzUwZWI4N2M3Yjc4","log_type":"infrastructure"} {"@timestamp":"2025-05-14T18:59:34.960729968+00:00","message":"W0514 18:59:34.960633 1 logging.go:59] [core] [Channel #33910 SubChannel #33911] grpc: addrConn.createTransport failed to connect to {","docker":{"container_id":"52fafb371b081b186f393608645eb0ebf994368572c863145322d23fd65652f4"},"kubernetes":{"container_name":"openshift-apiserver","namespace_name":"openshift-apiserver","pod_name":"apiserver-6bd8bfc7df-msbmm","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf3e4ed10a434395e26a0df443e8c41ec4db1cc64b4e4d291639da5db0a7ae95","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf3e4ed10a434395e26a0df443e8c41ec4db1cc64b4e4d291639da5db0a7ae95","pod_id":"8468ec72-57ed-448e-a6c8-11184d75e9a1","pod_ip":"10.128.1.43","host":"master-0","labels":{"apiserver":"true","app":"openshift-apiserver-a","openshift-apiserver-anti-affinity":"true","pod-template-hash":"6bd8bfc7df","revision":"1"},"master_url":"https://kubernetes.default.svc","namespace_id":"73810a40-c12b-492f-abd7-cd16f20f57f0","namespace_labels":{"kubernetes_io_metadata_name":"openshift-apiserver","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["apiserver=true","app=openshift-apiserver-a","openshift-apiserver-anti-affinity=true","pod-template-hash=6bd8bfc7df","revision=1"]},"level":"warn","hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.64","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:59:34.969394+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":77583,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-infra","logType":"infra"}},"viaq_msg_id":"N2VhZjIzOGUtMDBmNS00MmVjLTg5N2QtNjU3YWM2YWViM2Q1","log_type":"infrastructure"} {"@timestamp":"2025-05-14T18:59:34.960729968+00:00","message":" \"Addr\": \"10.20.177.159:2379\",","docker":{"container_id":"52fafb371b081b186f393608645eb0ebf994368572c863145322d23fd65652f4"},"kubernetes":{"container_name":"openshift-apiserver","namespace_name":"openshift-apiserver","pod_name":"apiserver-6bd8bfc7df-msbmm","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf3e4ed10a434395e26a0df443e8c41ec4db1cc64b4e4d291639da5db0a7ae95","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf3e4ed10a434395e26a0df443e8c41ec4db1cc64b4e4d291639da5db0a7ae95","pod_id":"8468ec72-57ed-448e-a6c8-11184d75e9a1","pod_ip":"10.128.1.43","host":"master-0","labels":{"apiserver":"true","app":"openshift-apiserver-a","openshift-apiserver-anti-affinity":"true","pod-template-hash":"6bd8bfc7df","revision":"1"},"master_url":"https://kubernetes.default.svc","namespace_id":"73810a40-c12b-492f-abd7-cd16f20f57f0","namespace_labels":{"kubernetes_io_metadata_name":"openshift-apiserver","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["apiserver=true","app=openshift-apiserver-a","openshift-apiserver-anti-affinity=true","pod-template-hash=6bd8bfc7df","revision=1"]},"level":"unknown","hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.64","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:59:34.989351+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":77584,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-infra","logType":"infra"}},"viaq_msg_id":"NmRkNWM2NDEtYWZiMy00ZmUxLTliZTMtYWY1MjZhZWQ2NDQ4","log_type":"infrastructure"} {"@timestamp":"2025-05-14T18:59:34.960729968+00:00","message":" \"ServerName\": \"10.20.177.159\",","docker":{"container_id":"52fafb371b081b186f393608645eb0ebf994368572c863145322d23fd65652f4"},"kubernetes":{"container_name":"openshift-apiserver","namespace_name":"openshift-apiserver","pod_name":"apiserver-6bd8bfc7df-msbmm","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf3e4ed10a434395e26a0df443e8c41ec4db1cc64b4e4d291639da5db0a7ae95","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf3e4ed10a434395e26a0df443e8c41ec4db1cc64b4e4d291639da5db0a7ae95","pod_id":"8468ec72-57ed-448e-a6c8-11184d75e9a1","pod_ip":"10.128.1.43","host":"master-0","labels":{"apiserver":"true","app":"openshift-apiserver-a","openshift-apiserver-anti-affinity":"true","pod-template-hash":"6bd8bfc7df","revision":"1"},"master_url":"https://kubernetes.default.svc","namespace_id":"73810a40-c12b-492f-abd7-cd16f20f57f0","namespace_labels":{"kubernetes_io_metadata_name":"openshift-apiserver","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["apiserver=true","app=openshift-apiserver-a","openshift-apiserver-anti-affinity=true","pod-template-hash=6bd8bfc7df","revision=1"]},"level":"unknown","hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.64","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:59:34.989823+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":77585,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-infra","logType":"infra"}},"viaq_msg_id":"ZGI0ZjhlYjktMThkOC00ODFiLWI0YjItN2NmYjBiMDI1MDEz","log_type":"infrastructure"} {"@timestamp":"2025-05-14T18:59:34.960729968+00:00","message":" \"Attributes\": null,","docker":{"container_id":"52fafb371b081b186f393608645eb0ebf994368572c863145322d23fd65652f4"},"kubernetes":{"container_name":"openshift-apiserver","namespace_name":"openshift-apiserver","pod_name":"apiserver-6bd8bfc7df-msbmm","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf3e4ed10a434395e26a0df443e8c41ec4db1cc64b4e4d291639da5db0a7ae95","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf3e4ed10a434395e26a0df443e8c41ec4db1cc64b4e4d291639da5db0a7ae95","pod_id":"8468ec72-57ed-448e-a6c8-11184d75e9a1","pod_ip":"10.128.1.43","host":"master-0","labels":{"apiserver":"true","app":"openshift-apiserver-a","openshift-apiserver-anti-affinity":"true","pod-template-hash":"6bd8bfc7df","revision":"1"},"master_url":"https://kubernetes.default.svc","namespace_id":"73810a40-c12b-492f-abd7-cd16f20f57f0","namespace_labels":{"kubernetes_io_metadata_name":"openshift-apiserver","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["apiserver=true","app=openshift-apiserver-a","openshift-apiserver-anti-affinity=true","pod-template-hash=6bd8bfc7df","revision=1"]},"level":"unknown","hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.64","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:59:34.990145+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":77586,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-infra","logType":"infra"}},"viaq_msg_id":"MmRlZDI0NjEtYzI4MC00YTQxLTg3NDYtNGExNjNlNjdlMzA0","log_type":"infrastructure"} {"@timestamp":"2025-05-14T18:59:34.960729968+00:00","message":" \"BalancerAttributes\": null,","docker":{"container_id":"52fafb371b081b186f393608645eb0ebf994368572c863145322d23fd65652f4"},"kubernetes":{"container_name":"openshift-apiserver","namespace_name":"openshift-apiserver","pod_name":"apiserver-6bd8bfc7df-msbmm","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf3e4ed10a434395e26a0df443e8c41ec4db1cc64b4e4d291639da5db0a7ae95","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf3e4ed10a434395e26a0df443e8c41ec4db1cc64b4e4d291639da5db0a7ae95","pod_id":"8468ec72-57ed-448e-a6c8-11184d75e9a1","pod_ip":"10.128.1.43","host":"master-0","labels":{"apiserver":"true","app":"openshift-apiserver-a","openshift-apiserver-anti-affinity":"true","pod-template-hash":"6bd8bfc7df","revision":"1"},"master_url":"https://kubernetes.default.svc","namespace_id":"73810a40-c12b-492f-abd7-cd16f20f57f0","namespace_labels":{"kubernetes_io_metadata_name":"openshift-apiserver","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["apiserver=true","app=openshift-apiserver-a","openshift-apiserver-anti-affinity=true","pod-template-hash=6bd8bfc7df","revision=1"]},"level":"unknown","hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.64","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T18:59:34.990382+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":77587,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"5.8.20_415-infra","logType":"infra"}},"viaq_msg_id":"NjM0YjQyZDUtOTMyNy00YmU4LWFkNjAtMTFjMWVkYzMwMmVm","log_type":"infrastructure"} ``` Loki: Application logs: ``` {"streams":[],"stats":{"summary":{"bytesProcessedPerSecond":0,"linesProcessedPerSecond":0,"totalBytesProcessed":0,"totalLinesProcessed":0,"execTime":0.005456257,"queueTime":0.00016,"subqueries":0,"totalEntriesReturned":0,"splits":3,"shards":0},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0}}},"ingester":{"totalReached":3,"totalChunksMatched":0,"totalBatches":0,"totalLinesSent":0,"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0}}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0}}}} ``` Audit logs: ``` {"ts":"2025-05-14T19:07:14.95399Z","line":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"a541f3f6-3c8f-4fb6-9673-697dfd802115\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/config.openshift.io/v1/clusteroperators/machine-config\",\"verb\":\"get\",\"user\":{\"username\":\"system:serviceaccount:openshift-machine-config-operator:machine-config-operator\",\"uid\":\"7a8bac61-5ed4-4f13-868d-f8258ea7799b\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-machine-config-operator\",\"system:authenticated\"],\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"machine-config-operator-59cd87895f-45skp\"],\"authentication.kubernetes.io/pod-uid\":[\"27f612b2-f93a-4d93-81f3-80559315b454\"]}},\"sourceIPs\":[\"10.129.1.61\"],\"userAgent\":\"machine-config-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format/machine-config\",\"objectRef\":{\"resource\":\"clusteroperators\",\"name\":\"machine-config\",\"apiGroup\":\"config.openshift.io\",\"apiVersion\":\"v1\"},\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2025-05-14T19:07:14.953990Z\",\"stageTimestamp\":\"2025-05-14T19:07:14.956870Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"custom-account-openshift-machine-config-operator\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"machine-config-operator/openshift-machine-config-operator\\\"\"},\"@timestamp\":\"2025-05-14T19:07:14.953990Z\",\"k8s_audit_level\":\"Metadata\",\"message\":null,\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.177.159\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-05-14T19:07:14.957195+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":12806,\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"logs\":\"5.8_415-audit\"}},\"viaq_msg_id\":\"ZTM1ODAzOWItMDU0ZS00YTI2LTgzMmYtYjAxMDQxYjM5MzQ2\",\"log_type\":\"audit\"}"},{"ts":"2025-05-14T19:07:14.946478Z","line":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"76caa9a6-ca2c-499a-bcd0-34f7eb4620fb\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/config.openshift.io/v1/clusteroperators/machine-config/status\",\"verb\":\"update\",\"user\":{\"username\":\"system:serviceaccount:openshift-machine-config-operator:machine-config-operator\",\"uid\":\"7a8bac61-5ed4-4f13-868d-f8258ea7799b\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-machine-config-operator\",\"system:authenticated\"],\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"machine-config-operator-59cd87895f-45skp\"],\"authentication.kubernetes.io/pod-uid\":[\"27f612b2-f93a-4d93-81f3-80559315b454\"]}},\"sourceIPs\":[\"10.129.1.61\"],\"userAgent\":\"machine-config-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format/machine-config\",\"objectRef\":{\"resource\":\"clusteroperators\",\"name\":\"machine-config\",\"uid\":\"25aea5e3-26f9-4fb6-8283-e38628c8dba1\",\"apiGroup\":\"config.openshift.io\",\"apiVersion\":\"v1\",\"resourceVersion\":\"5631238\",\"subresource\":\"status\"},\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2025-05-14T19:07:14.946478Z\",\"stageTimestamp\":\"2025-05-14T19:07:14.952980Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"custom-account-openshift-machine-config-operator\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"machine-config-operator/openshift-machine-config-operator\\\"\"},\"@timestamp\":\"2025-05-14T19:07:14.946478Z\",\"k8s_audit_level\":\"Metadata\",\"message\":null,\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.177.159\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-05-14T19:07:14.953416+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":12805,\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"logs\":\"5.8_415-audit\"}},\"viaq_msg_id\":\"OWYyZDE4MjQtYzE3Yi00MWE5LWI3ODQtNzc2MTcxYTU4OGUw\",\"log_type\":\"audit\"}"},{"ts":"2025-05-14T19:07:14.942353Z","line":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"f70968c4-8c6d-4af0-8ba1-0276dccdd8ce\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/config.openshift.io/v1/clusteroperators/machine-config\",\"verb\":\"get\",\"user\":{\"username\":\"system:serviceaccount:openshift-machine-config-operator:machine-config-operator\",\"uid\":\"7a8bac61-5ed4-4f13-868d-f8258ea7799b\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-machine-config-operator\",\"system:authenticated\"],\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"machine-config-operator-59cd87895f-45skp\"],\"authentication.kubernetes.io/pod-uid\":[\"27f612b2-f93a-4d93-81f3-80559315b454\"]}},\"sourceIPs\":[\"10.129.1.61\"],\"userAgent\":\"machine-config-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format/machine-config\",\"objectRef\":{\"resource\":\"clusteroperators\",\"name\":\"machine-config\",\"apiGroup\":\"config.openshift.io\",\"apiVersion\":\"v1\"},\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2025-05-14T19:07:14.942353Z\",\"stageTimestamp\":\"2025-05-14T19:07:14.945353Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"custom-account-openshift-machine-config-operator\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"machine-config-operator/openshift-machine-config-operator\\\"\"},\"@timestamp\":\"2025-05-14T19:07:14.942353Z\",\"k8s_audit_level\":\"Metadata\",\"message\":null,\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.177.159\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-05-14T19:07:14.945723+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":12804,\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"logs\":\"5.8_415-audit\"}},\"viaq_msg_id\":\"MTg3YWI2N2QtMmU1Yi00NzIyLWIxM2EtMDM2ZjVlMWRlOTc2\",\"log_type\":\"audit\"}"},{"ts":"2025-05-14T19:07:14.932095Z","line":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"6fac3d2c-6265-4371-aacb-b2786c8cbb6d\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/config.openshift.io/v1/clusteroperators/machine-config/status\",\"verb\":\"update\",\"user\":{\"username\":\"system:serviceaccount:openshift-machine-config-operator:machine-config-operator\",\"uid\":\"7a8bac61-5ed4-4f13-868d-f8258ea7799b\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-machine-config-operator\",\"system:authenticated\"],\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"machine-config-operator-59cd87895f-45skp\"],\"authentication.kubernetes.io/pod-uid\":[\"27f612b2-f93a-4d93-81f3-80559315b454\"]}},\"sourceIPs\":[\"10.129.1.61\"],\"userAgent\":\"machine-config-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format/machine-config\",\"objectRef\":{\"resource\":\"clusteroperators\",\"name\":\"machine-config\",\"uid\":\"25aea5e3-26f9-4fb6-8283-e38628c8dba1\",\"apiGroup\":\"config.openshift.io\",\"apiVersion\":\"v1\",\"resourceVersion\":\"5631238\",\"subresource\":\"status\"},\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2025-05-14T19:07:14.932095Z\",\"stageTimestamp\":\"2025-05-14T19:07:14.941092Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"custom-account-openshift-machine-config-operator\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"machine-config-operator/openshift-machine-config-operator\\\"\"},\"@timestamp\":\"2025-05-14T19:07:14.932095Z\",\"k8s_audit_level\":\"Metadata\",\"message\":null,\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.177.159\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-05-14T19:07:14.941637+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":12803,\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"logs\":\"5.8_415-audit\"}},\"viaq_msg_id\":\"ZjkzNTE5ZTYtYmZjYS00ZDJiLWFjMWItNjQ4Yzk4ZDJhYjNk\",\"log_type\":\"audit\"}"},{"ts":"2025-05-14T19:07:14.926197Z","line":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"d2a64df0-46b4-4446-86e3-da34fed98539\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/config.openshift.io/v1/clusteroperators/machine-config\",\"verb\":\"get\",\"user\":{\"username\":\"system:serviceaccount:openshift-machine-config-operator:machine-config-operator\",\"uid\":\"7a8bac61-5ed4-4f13-868d-f8258ea7799b\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-machine-config-operator\",\"system:authenticated\"],\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"machine-config-operator-59cd87895f-45skp\"],\"authentication.kubernetes.io/pod-uid\":[\"27f612b2-f93a-4d93-81f3-80559315b454\"]}},\"sourceIPs\":[\"10.129.1.61\"],\"userAgent\":\"machine-config-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format/machine-config\",\"objectRef\":{\"resource\":\"clusteroperators\",\"name\":\"machine-config\",\"apiGroup\":\"config.openshift.io\",\"apiVersion\":\"v1\"},\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2025-05-14T19:07:14.926197Z\",\"stageTimestamp\":\"2025-05-14T19:07:14.930687Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"custom-account-openshift-machine-config-operator\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"machine-config-operator/openshift-machine-config-operator\\\"\"},\"@timestamp\":\"2025-05-14T19:07:14.926197Z\",\"k8s_audit_level\":\"Metadata\",\"message\":null,\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.177.159\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-05-14T19:07:14.931182+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":12802,\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"logs\":\"5.8_415-audit\"}},\"viaq_msg_id\":\"MmYxMDU1ZmQtMDMzNy00MTcyLWExMzAtZjFmNTMwNzdiYjM1\",\"log_type\":\"audit\"}"},{"ts":"2025-05-14T19:07:14.920872Z","line":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"c4fb2822-9907-4967-ab49-5747e2073a5d\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/config.openshift.io/v1/clusteroperators/machine-config\",\"verb\":\"get\",\"user\":{\"username\":\"system:serviceaccount:openshift-machine-config-operator:machine-config-operator\",\"uid\":\"7a8bac61-5ed4-4f13-868d-f8258ea7799b\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-machine-config-operator\",\"system:authenticated\"],\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"machine-config-operator-59cd87895f-45skp\"],\"authentication.kubernetes.io/pod-uid\":[\"27f612b2-f93a-4d93-81f3-80559315b454\"]}},\"sourceIPs\":[\"10.129.1.61\"],\"userAgent\":\"machine-config-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format/machine-config\",\"objectRef\":{\"resource\":\"clusteroperators\",\"name\":\"machine-config\",\"apiGroup\":\"config.openshift.io\",\"apiVersion\":\"v1\"},\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2025-05-14T19:07:14.920872Z\",\"stageTimestamp\":\"2025-05-14T19:07:14.924531Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"custom-account-openshift-machine-config-operator\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"machine-config-operator/openshift-machine-config-operator\\\"\"},\"@timestamp\":\"2025-05-14T19:07:14.920872Z\",\"k8s_audit_level\":\"Metadata\",\"message\":null,\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.177.159\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-05-14T19:07:14.925400+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":12801,\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"logs\":\"5.8_415-audit\"}},\"viaq_msg_id\":\"YmM3NDE3YzItZjlmNy00Y2Q0LWJiY2QtY2JmOTkyNjgwNDZh\",\"log_type\":\"audit\"}"},{"ts":"2025-05-14T19:07:14.827746Z","line":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"fbbb5cd2-d664-4900-9200-6a90984923c0\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/authorization.k8s.io/v1/subjectaccessreviews\",\"verb\":\"create\",\"user\":{\"username\":\"system:serviceaccount:openshift-monitoring:alertmanager-main\",\"uid\":\"299b4463-0f97-4b9f-a3a5-57be4e9a0588\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-monitoring\",\"system:authenticated\"],\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"alertmanager-main-1\"],\"authentication.kubernetes.io/pod-uid\":[\"80c67631-3fbe-4103-9c54-b2f851d4e4a5\"]}},\"sourceIPs\":[\"10.20.177.189\"],\"userAgent\":\"oauth-proxy/v0.0.0 (linux/ppc64le) kubernetes/$Format\",\"objectRef\":{\"resource\":\"subjectaccessreviews\",\"apiGroup\":\"authorization.k8s.io\",\"apiVersion\":\"v1\"},\"responseStatus\":{\"metadata\":{},\"code\":201},\"requestReceivedTimestamp\":\"2025-05-14T19:07:14.827746Z\",\"stageTimestamp\":\"2025-05-14T19:07:14.829249Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"alertmanager-main\\\" of ClusterRole \\\"alertmanager-main\\\" to ServiceAccount \\\"alertmanager-main/openshift-monitoring\\\"\"},\"@timestamp\":\"2025-05-14T19:07:14.827746Z\",\"k8s_audit_level\":\"Metadata\",\"message\":null,\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.177.159\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-05-14T19:07:14.829531+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":12798,\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"logs\":\"5.8_415-audit\"}},\"viaq_msg_id\":\"MWZmNjNhY2EtNmMzNi00YmVhLTk3OWUtNGU1MjY0ODc4MzE4\",\"log_type\":\"audit\"}"}]}]} ``` Infrastructure logs: ``` {"ts":"2025-05-14T19:07:00.094462869Z","line":"{\"@timestamp\":\"2025-05-14T19:07:00.094462869+00:00\",\"message\":\"I0514 19:07:00.094391 1 log.go:194] reconciling (monitoring.coreos.com/v1, Kind=PrometheusRule) openshift-ovn-kubernetes/master-rules\",\"docker\":{\"container_id\":\"106a6ca04bf159cbd7800d42c0d7224da4ae75733679c31ae830b749e4cbe380\"},\"kubernetes\":{\"container_name\":\"network-operator\",\"namespace_name\":\"openshift-network-operator\",\"pod_name\":\"network-operator-768fcf7769-xzl99\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bd9a861749b48f743a5e9e6184fbc0b7fcb0011c5759755f3ff85f4ea383b0f\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bd9a861749b48f743a5e9e6184fbc0b7fcb0011c5759755f3ff85f4ea383b0f\",\"pod_id\":\"c268c47e-c067-44a5-b366-34cd85e7e54f\",\"pod_ip\":\"10.20.177.159\",\"host\":\"master-1\",\"labels\":{\"name\":\"network-operator\",\"pod-template-hash\":\"768fcf7769\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"bdcb252c-d252-4e2a-a7ac-6579f32c386c\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-network-operator\",\"name\":\"openshift-network-operator\",\"olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"openshift_io_run-level\":\"0\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"flat_labels\":[\"name=network-operator\",\"pod-template-hash=768fcf7769\"]},\"level\":\"info\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.177.159\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-05-14T19:07:00.095341+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":12333,\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"logs\":\"5.8_415-infrastructure\"}},\"viaq_msg_id\":\"YjdiZWMyNWUtMGNkYS00M2YxLWJmODAtNWJkZTJiM2U3OWY0\",\"log_type\":\"infrastructure\"}"},{"ts":"2025-05-14T19:07:00.09438766Z","line":"{\"@timestamp\":\"2025-05-14T19:07:00.094387660+00:00\",\"message\":\"I0514 19:07:00.094342 1 log.go:194] Apply / Create of (/v1, Kind=ConfigMap) openshift-ovn-kubernetes/ovnkube-script-lib was successful\",\"docker\":{\"container_id\":\"106a6ca04bf159cbd7800d42c0d7224da4ae75733679c31ae830b749e4cbe380\"},\"kubernetes\":{\"container_name\":\"network-operator\",\"namespace_name\":\"openshift-network-operator\",\"pod_name\":\"network-operator-768fcf7769-xzl99\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bd9a861749b48f743a5e9e6184fbc0b7fcb0011c5759755f3ff85f4ea383b0f\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bd9a861749b48f743a5e9e6184fbc0b7fcb0011c5759755f3ff85f4ea383b0f\",\"pod_id\":\"c268c47e-c067-44a5-b366-34cd85e7e54f\",\"pod_ip\":\"10.20.177.159\",\"host\":\"master-1\",\"labels\":{\"name\":\"network-operator\",\"pod-template-hash\":\"768fcf7769\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"bdcb252c-d252-4e2a-a7ac-6579f32c386c\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-network-operator\",\"name\":\"openshift-network-operator\",\"olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"openshift_io_run-level\":\"0\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"flat_labels\":[\"name=network-operator\",\"pod-template-hash=768fcf7769\"]},\"level\":\"info\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.177.159\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-05-14T19:07:00.095151+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":12332,\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"logs\":\"5.8_415-infrastructure\"}},\"viaq_msg_id\":\"YzI0OWFkZTctOGJlMC00ODk0LTg2YzMtMTY3MjY4ZDEyYTBi\",\"log_type\":\"infrastructure\"}"},{"ts":"2025-05-14T19:07:00.058834928Z","line":"{\"@timestamp\":\"2025-05-14T19:07:00.058834928+00:00\",\"message\":\"I0514 19:07:00.058784 1 log.go:194] The check PodNetworkConnectivityCheck/network-check-source-worker-0-to-network-check-target-master-0 -n openshift-network-diagnostics is applied\",\"docker\":{\"container_id\":\"106a6ca04bf159cbd7800d42c0d7224da4ae75733679c31ae830b749e4cbe380\"},\"kubernetes\":{\"container_name\":\"network-operator\",\"namespace_name\":\"openshift-network-operator\",\"pod_name\":\"network-operator-768fcf7769-xzl99\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bd9a861749b48f743a5e9e6184fbc0b7fcb0011c5759755f3ff85f4ea383b0f\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bd9a861749b48f743a5e9e6184fbc0b7fcb0011c5759755f3ff85f4ea383b0f\",\"pod_id\":\"c268c47e-c067-44a5-b366-34cd85e7e54f\",\"pod_ip\":\"10.20.177.159\",\"host\":\"master-1\",\"labels\":{\"name\":\"network-operator\",\"pod-template-hash\":\"768fcf7769\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"bdcb252c-d252-4e2a-a7ac-6579f32c386c\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-network-operator\",\"name\":\"openshift-network-operator\",\"olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"openshift_io_run-level\":\"0\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"flat_labels\":[\"name=network-operator\",\"pod-template-hash=768fcf7769\"]},\"level\":\"info\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.177.159\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-05-14T19:07:00.059681+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":12331,\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"logs\":\"5.8_415-infrastructure\"}},\"viaq_msg_id\":\"ZDBlZjk2MmItMTgzOS00ZTRhLTkzNDYtYmIyZDc2NWFlOGU4\",\"log_type\":\"infrastructure\"}"},{"ts":"2025-05-14T19:06:59.836426699Z","line":"{\"@timestamp\":\"2025-05-14T19:06:59.836426699+00:00\",\"message\":\"I0514 19:06:59.836371 1 log.go:194] The check PodNetworkConnectivityCheck/network-check-source-worker-0-to-network-check-target-service-cluster -n openshift-network-diagnostics is applied\",\"docker\":{\"container_id\":\"106a6ca04bf159cbd7800d42c0d7224da4ae75733679c31ae830b749e4cbe380\"},\"kubernetes\":{\"container_name\":\"network-operator\",\"namespace_name\":\"openshift-network-operator\",\"pod_name\":\"network-operator-768fcf7769-xzl99\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bd9a861749b48f743a5e9e6184fbc0b7fcb0011c5759755f3ff85f4ea383b0f\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bd9a861749b48f743a5e9e6184fbc0b7fcb0011c5759755f3ff85f4ea383b0f\",\"pod_id\":\"c268c47e-c067-44a5-b366-34cd85e7e54f\",\"pod_ip\":\"10.20.177.159\",\"host\":\"master-1\",\"labels\":{\"name\":\"network-operator\",\"pod-template-hash\":\"768fcf7769\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"bdcb252c-d252-4e2a-a7ac-6579f32c386c\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-network-operator\",\"name\":\"openshift-network-operator\",\"olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"openshift_io_run-level\":\"0\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"flat_labels\":[\"name=network-operator\",\"pod-template-hash=768fcf7769\"]},\"level\":\"info\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.177.159\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-05-14T19:06:59.837147+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":12323,\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"logs\":\"5.8_415-infrastructure\"}},\"viaq_msg_id\":\"NzBjMmJiMjEtM2E1YS00NTQ2LTgwZjMtNWRkMzFiMWQ2YTVm\",\"log_type\":\"infrastructure\"}"},{"ts":"2025-05-14T19:06:59.667704265Z","line":"{\"@timestamp\":\"2025-05-14T19:06:59.667704265+00:00\",\"message\":\"I0514 19:06:59.667670 1 log.go:194] reconciling (/v1, Kind=ConfigMap) openshift-ovn-kubernetes/ovnkube-script-lib\",\"docker\":{\"container_id\":\"106a6ca04bf159cbd7800d42c0d7224da4ae75733679c31ae830b749e4cbe380\"},\"kubernetes\":{\"container_name\":\"network-operator\",\"namespace_name\":\"openshift-network-operator\",\"pod_name\":\"network-operator-768fcf7769-xzl99\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bd9a861749b48f743a5e9e6184fbc0b7fcb0011c5759755f3ff85f4ea383b0f\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bd9a861749b48f743a5e9e6184fbc0b7fcb0011c5759755f3ff85f4ea383b0f\",\"pod_id\":\"c268c47e-c067-44a5-b366-34cd85e7e54f\",\"pod_ip\":\"10.20.177.159\",\"host\":\"master-1\",\"labels\":{\"name\":\"network-operator\",\"pod-template-hash\":\"768fcf7769\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"bdcb252c-d252-4e2a-a7ac-6579f32c386c\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-network-operator\",\"name\":\"openshift-network-operator\",\"olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"openshift_io_run-level\":\"0\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"flat_labels\":[\"name=network-operator\",\"pod-template-hash=768fcf7769\"]},\"level\":\"info\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.177.159\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-05-14T19:06:59.669003+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":12321,\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"logs\":\"5.8_415-infrastructure\"}},\"viaq_msg_id\":\"NDNkMTVmZjItZDBkYS00NGQ4LThkNmItMGUxNTZmMjAxM2Ey\",\"log_type\":\"infrastructure\"}"}]},{"labels":"{fluentd_thread=\"flush_thread_0\", kubernetes_container_name=\"insights-operator\", kubernetes_host=\"master-1\", kubernetes_namespace_name=\"openshift-insights\", kubernetes_pod_name=\"insights-operator-7d6b86d874-jwcgm\", log_type=\"infrastructure\"}","entries":[{"ts":"2025-05-14T19:07:17.742942746Z","line":"{\"@timestamp\":\"2025-05-14T19:07:17.742942746+00:00\",\"message\":\"I0514 19:07:17.742883 1 httplog.go:132] \\\"HTTP\\\" verb=\\\"GET\\\" URI=\\\"/metrics\\\" latency=\\\"6.72841ms\\\" userAgent=\\\"Prometheus/2.48.0\\\" audit-ID=\\\"9ab7c2ab-0de0-4ff1-b971-ad7f57a11765\\\" srcIP=\\\"10.128.3.146:34010\\\" resp=200\",\"docker\":{\"container_id\":\"cf68cf2e5dec5f4d728a80df73bdae193163ce185c2abc7568697aa725385bba\"},\"kubernetes\":{\"container_name\":\"insights-operator\",\"namespace_name\":\"openshift-insights\",\"pod_name\":\"insights-operator-7d6b86d874-jwcgm\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8668c4346983fe645317e7c4f53e6de9b4246271e21f8a55b3c5eb7fe5459fd5\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8668c4346983fe645317e7c4f53e6de9b4246271e21f8a55b3c5eb7fe5459fd5\",\"pod_id\":\"e071027b-21f9-4f3c-ae58-e05e4628b7ff\",\"pod_ip\":\"10.129.1.62\",\"host\":\"master-1\",\"labels\":{\"app\":\"insights-operator\",\"pod-template-hash\":\"7d6b86d874\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"cab8f97f-577c-47e3-af83-e3577f9ea867\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-insights\",\"name\":\"openshift-insights\",\"olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"openshift_io_run-level\":\"\"},\"flat_labels\":[\"app=insights-operator\",\"pod-template-hash=7d6b86d874\"]},\"level\":\"info\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.177.159\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-05-14T19:07:17.743935+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":12852,\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"logs\":\"5.8_415-infrastructure\"}},\"viaq_msg_id\":\"OTBlNzUzOWYtNTE5MC00OGYzLWJlNzAtMjNmNTBiZmFmNjI2\",\"log_type\":\"infrastructure\"}"}]}],"stats":{"summary":{"bytesProcessedPerSecond":91342437,"linesProcessedPerSecond":47097,"totalBytesProcessed":1055049,"totalLinesProcessed":544,"execTime":0.01155048,"queueTime":0.000105,"subqueries":0,"totalEntriesReturned":100,"splits":1,"shards":0},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0}}},"ingester":{"totalReached":1,"totalChunksMatched":23,"totalBatches":1,"totalLinesSent":100,"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":1044683,"headChunkLines":538,"decompressedBytes":10366,"decompressedLines":6,"compressedBytes":33935,"totalDuplicates":0}}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0}}}} ``` Fluentd: Application logs: ``` 2025-05-14T19:18:11+00:00 kubernetes.var.log.pods.test-logging_centos-logtest-bn5bw_deb41613-5199-4569-8d78-7cb401a303f9.centos-logtest.0.log {"@timestamp":"2025-05-14T19:18:11.293384612+00:00","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","docker":{"container_id":"d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"test-logging","pod_name":"centos-logtest-bn5bw","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"deb41613-5199-4569-8d78-7cb401a303f9","pod_ip":"10.131.0.195","host":"worker-0","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"unknown","hostname":"worker-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T19:18:11.295773+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":350,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"clusterId":"ocp47-log507-19","logs":"5.8.20_415_app"}},"viaq_msg_id":"MjY2Yzg1NmUtYTY4YS00YTZjLTgxYzMtNWUxM2EyZThjYzNm","log_type":"application"} 2025-05-14T19:18:12+00:00 kubernetes.var.log.pods.test-logging_centos-logtest-bn5bw_deb41613-5199-4569-8d78-7cb401a303f9.centos-logtest.0.log {"@timestamp":"2025-05-14T19:18:12.294917090+00:00","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","docker":{"container_id":"d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"test-logging","pod_name":"centos-logtest-bn5bw","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"deb41613-5199-4569-8d78-7cb401a303f9","pod_ip":"10.131.0.195","host":"worker-0","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"unknown","hostname":"worker-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T19:18:12.297052+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":351,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"clusterId":"ocp47-log507-19","logs":"5.8.20_415_app"}},"viaq_msg_id":"ZDAzMThjNmMtOGY4Zi00NDIyLWE0ODYtOWNmMGMwZDY1ZGFm","log_type":"application"} 2025-05-14T19:18:13+00:00 kubernetes.var.log.pods.test-logging_centos-logtest-bn5bw_deb41613-5199-4569-8d78-7cb401a303f9.centos-logtest.0.log {"@timestamp":"2025-05-14T19:18:13.295675678+00:00","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","docker":{"container_id":"d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"test-logging","pod_name":"centos-logtest-bn5bw","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"deb41613-5199-4569-8d78-7cb401a303f9","pod_ip":"10.131.0.195","host":"worker-0","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"unknown","hostname":"worker-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T19:18:13.298376+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":352,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"clusterId":"ocp47-log507-19","logs":"5.8.20_415_app"}},"viaq_msg_id":"MTRmMzIyZmMtMTJkZi00NDkyLTkyNDMtNTAzMWVhYzIzNzYw","log_type":"application"} 2025-05-14T19:18:14+00:00 kubernetes.var.log.pods.test-logging_centos-logtest-bn5bw_deb41613-5199-4569-8d78-7cb401a303f9.centos-logtest.0.log {"@timestamp":"2025-05-14T19:18:14.297148506+00:00","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","docker":{"container_id":"d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"test-logging","pod_name":"centos-logtest-bn5bw","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"deb41613-5199-4569-8d78-7cb401a303f9","pod_ip":"10.131.0.195","host":"worker-0","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"unknown","hostname":"worker-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T19:18:14.298630+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":355,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"clusterId":"ocp47-log507-19","logs":"5.8.20_415_app"}},"viaq_msg_id":"NTAwNTAzZTItMTkzOC00NmMxLWIyZTktYzEzNTdhNjBhZjU0","log_type":"application"} ``` Audit logs: ``` 2025-05-14T19:21:54+00:00 k8s-audit.log {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"83c8c0e7-ddb4-4465-bb89-b5160fef875c","stage":"ResponseComplete","requestURI":"/apis/authorization.k8s.io/v1/subjectaccessreviews?timeout=10s","verb":"create","user":{"username":"system:serviceaccount:openshift-apiserver:openshift-apiserver-sa","uid":"ec49b816-ebc9-4fed-8745-be7929678376","groups":["system:serviceaccounts","system:serviceaccounts:openshift-apiserver","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["apiserver-6bd8bfc7df-pvzz6"],"authentication.kubernetes.io/pod-uid":["8db37b33-7067-4661-87d4-b2193acdbb5d"]}},"sourceIPs":["10.20.177.77"],"userAgent":"openshift-apiserver/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"subjectaccessreviews","apiGroup":"authorization.k8s.io","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":201},"requestReceivedTimestamp":"2025-05-14T19:21:54.790558Z","stageTimestamp":"2025-05-14T19:21:54.795022Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:openshift:openshift-apiserver\" of ClusterRole \"cluster-admin\" to ServiceAccount \"openshift-apiserver-sa/openshift-apiserver\""},"@timestamp":"2025-05-14T19:21:54.790558Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.64","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T19:21:54.798120+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":26128,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"clusterId":"ocp47-log507-19","logs":"5.8.20_415_audit"}},"viaq_msg_id":"MDRkYTgwZGUtZmQwMi00ODNjLTgwOWYtMmZkMDM1ZTQ5NzBl","log_type":"audit"} 2025-05-14T19:21:54+00:00 k8s-audit.log {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"bc9a5874-2adc-46fd-8983-a2c982eff85f","stage":"ResponseComplete","requestURI":"/apis/logging.openshift.io/v1/namespaces/openshift-logging/elasticsearches/elasticsearch/status","verb":"update","user":{"username":"system:serviceaccount:openshift-operators-redhat:elasticsearch-operator","uid":"77fbbf3c-16c8-4c36-bd7b-a57817c5ea17","groups":["system:serviceaccounts","system:serviceaccounts:openshift-operators-redhat","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["elasticsearch-operator-54fdcdfb47-rrv2t"],"authentication.kubernetes.io/pod-uid":["dedfe19e-617a-4fb8-bbe6-90f14d8051fb"]}},"sourceIPs":["10.20.177.189"],"userAgent":"elasticsearch-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"elasticsearches","namespace":"openshift-logging","name":"elasticsearch","uid":"75a0fd37-73ec-4c34-8f2b-974c03247b49","apiGroup":"logging.openshift.io","apiVersion":"v1","resourceVersion":"5885553","subresource":"status"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-05-14T19:21:54.795638Z","stageTimestamp":"2025-05-14T19:21:54.807436Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"elasticsearch-operator.v-8yVvKvYJWxi8HPN4l2RIOONrojEfQtEzU5DfVO\" of ClusterRole \"elasticsearch-operator.v-8yVvKvYJWxi8HPN4l2RIOONrojEfQtEzU5DfVO\" to ServiceAccount \"elasticsearch-operator/openshift-operators-redhat\""},"@timestamp":"2025-05-14T19:21:54.795638Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.64","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T19:21:54.807804+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":26129,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"clusterId":"ocp47-log507-19","logs":"5.8.20_415_audit"}},"viaq_msg_id":"OGUyNDgzN2QtMzFhMC00ZGMyLThkYjctNmNjZWFiY2UyODlm","log_type":"audit"} 2025-05-14T19:21:54+00:00 k8s-audit.log {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"b5d10ada-4e4b-4d40-8bbe-f7e74301f666","stage":"ResponseComplete","requestURI":"/apis/monitoring.coreos.com/v1/namespaces/openshift-logging/servicemonitors/monitor-elasticsearch-cluster","verb":"update","user":{"username":"system:serviceaccount:openshift-operators-redhat:elasticsearch-operator","uid":"77fbbf3c-16c8-4c36-bd7b-a57817c5ea17","groups":["system:serviceaccounts","system:serviceaccounts:openshift-operators-redhat","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["elasticsearch-operator-54fdcdfb47-rrv2t"],"authentication.kubernetes.io/pod-uid":["dedfe19e-617a-4fb8-bbe6-90f14d8051fb"]}},"sourceIPs":["10.20.177.189"],"userAgent":"elasticsearch-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"servicemonitors","namespace":"openshift-logging","name":"monitor-elasticsearch-cluster","uid":"4696b637-2e88-42ed-95ba-ca006e20e449","apiGroup":"monitoring.coreos.com","apiVersion":"v1","resourceVersion":"5721373"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-05-14T19:21:54.810258Z","stageTimestamp":"2025-05-14T19:21:54.815811Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"elasticsearch-operator.v-8yVvKvYJWxi8HPN4l2RIOONrojEfQtEzU5DfVO\" of ClusterRole \"elasticsearch-operator.v-8yVvKvYJWxi8HPN4l2RIOONrojEfQtEzU5DfVO\" to ServiceAccount \"elasticsearch-operator/openshift-operators-redhat\""},"@timestamp":"2025-05-14T19:21:54.810258Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.64","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T19:21:54.816214+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":26130,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"clusterId":"ocp47-log507-19","logs":"5.8.20_415_audit"}},"viaq_msg_id":"MThlYzMzZmUtYWE0YS00YjA0LWE4MTYtZjUwZDU2YTNkZGI2","log_type":"audit"} 2025-05-14T19:21:54+00:00 k8s-audit.log {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"5cfb2ed4-4af8-436f-b495-61eabf6ac9ae","stage":"ResponseComplete","requestURI":"/apis/monitoring.coreos.com/v1/namespaces/openshift-logging/prometheusrules/elasticsearch-prometheus-rules","verb":"update","user":{"username":"system:serviceaccount:openshift-operators-redhat:elasticsearch-operator","uid":"77fbbf3c-16c8-4c36-bd7b-a57817c5ea17","groups":["system:serviceaccounts","system:serviceaccounts:openshift-operators-redhat","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["elasticsearch-operator-54fdcdfb47-rrv2t"],"authentication.kubernetes.io/pod-uid":["dedfe19e-617a-4fb8-bbe6-90f14d8051fb"]}},"sourceIPs":["10.20.177.189"],"userAgent":"elasticsearch-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"prometheusrules","namespace":"openshift-logging","name":"elasticsearch-prometheus-rules","uid":"318ed663-063f-42dd-becb-c0bfae1babe4","apiGroup":"monitoring.coreos.com","apiVersion":"v1","resourceVersion":"5721376"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-05-14T19:21:54.820533Z","stageTimestamp":"2025-05-14T19:21:54.840422Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"elasticsearch-operator.v-8yVvKvYJWxi8HPN4l2RIOONrojEfQtEzU5DfVO\" of ClusterRole \"elasticsearch-operator.v-8yVvKvYJWxi8HPN4l2RIOONrojEfQtEzU5DfVO\" to ServiceAccount \"elasticsearch-operator/openshift-operators-redhat\""},"@timestamp":"2025-05-14T19:21:54.820533Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.64","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T19:21:54.840840+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":26131,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"clusterId":"ocp47-log507-19","logs":"5.8.20_415_audit"}},"viaq_msg_id":"MWIxM2I4MDctNjM0YS00NTcyLWI5ZTItNzAwY2NjZWY1ZGRm","log_type":"audit"} ``` Infrastructure logs: ``` 2025-05-14T19:26:42+00:00 kubernetes.var.log.pods.openshift-kube-apiserver_kube-apiserver-master-1_bf98889dede239440ce9bd8be897279a.kube-apiserver.0.log {"@timestamp":"2025-05-14T19:26:42.285560246+00:00","message":"I0514 19:26:42.285451 18 httplog.go:92] &{system:serviceaccount:openshift-insights:operator 7e09a8c4-a3e5-4910-b55b-9e8da30bcfb2 [system:serviceaccounts system:serviceaccounts:openshift-insights system:authenticated] map[authentication.kubernetes.io/pod-name:[insights-operator-7d6b86d874-jwcgm] authentication.kubernetes.io/pod-uid:[e071027b-21f9-4f3c-ae58-e05e4628b7ff]]} is acting as &{system:serviceaccount:openshift-insights:gather [system:serviceaccounts system:serviceaccounts:openshift-insights system:authenticated] map[]}","docker":{"container_id":"bed89c5a6d5118734233cf4136c98318c780b82c8a4409bcf261e597932fbba5"},"kubernetes":{"container_name":"kube-apiserver","namespace_name":"openshift-kube-apiserver","pod_name":"kube-apiserver-master-1","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fea1716dc544df53dfdad9ba8c8402494195cde8b6eb3b4c95975dc158a12e9a","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fea1716dc544df53dfdad9ba8c8402494195cde8b6eb3b4c95975dc158a12e9a","pod_id":"47a62863-f500-467d-921c-bc95fce3c8c9","pod_ip":"10.20.177.159","host":"master-1","labels":{"apiserver":"true","app":"openshift-kube-apiserver","revision":"15"},"master_url":"https://kubernetes.default.svc","namespace_id":"3cb4e6a8-7c5f-4db1-8e7e-607248006ad5","namespace_labels":{"kubernetes_io_metadata_name":"openshift-kube-apiserver","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["apiserver=true","app=openshift-kube-apiserver","revision=15"]},"level":"info","hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.159","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T19:26:42.286239+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2356,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"clusterId":"ocp47-log507-19","logs":"5.8.20_415_infra"}},"viaq_msg_id":"YTMwNDA5MDgtMTNmZS00YzA0LTljZTAtZDMwMWE2ZWUzYTFk","log_type":"infrastructure"} 2025-05-14T19:26:42+00:00 journal.system {"_RUNTIME_SCOPE":"system","_SYSTEMD_INVOCATION_ID":"d4d5449bfc454bea8ea012d7f26127c9","SYSLOG_TIMESTAMP":"May 14 19:26:42 ","systemd":{"t":{"BOOT_ID":"4586239920664feb9666ba4c8e16d66c","CAP_EFFECTIVE":"7c00","CMDLINE":"ovs-vswitchd unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --user openvswitch:hugetlbfs --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach","COMM":"ovs-vswitchd","EXE":"/usr/sbin/ovs-vswitchd","GID":"801","MACHINE_ID":"391a7bcf45e544029be895a9654ed7ee","PID":"1122","SELINUX_CONTEXT":"system_u:system_r:openvswitch_t:s0","SYSTEMD_CGROUP":"/system.slice/ovs-vswitchd.service","SYSTEMD_INVOCATION_ID":"d4d5449bfc454bea8ea012d7f26127c9","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"ovs-vswitchd.service","TRANSPORT":"syslog","UID":"800"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"ovs-vswitchd"}},"level":"notice","message":"ovs|17042|connmgr|INFO|br-int<->unix#21: 3 flow_mods 12 s ago (3 adds)","hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.64","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T19:26:43.665597+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":2969,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"clusterId":"ocp47-log507-19","logs":"5.8.20_415_infra"}},"@timestamp":"2025-05-14T19:26:42.824873+00:00","viaq_msg_id":"NWQ5MzcwZWItNGYyMy00NmQxLWFhMDUtNzQ4ZmI0MTU1NzJk","log_type":"infrastructure"} 2025-05-14T19:26:44+00:00 kubernetes.var.log.pods.openshift-monitoring_thanos-querier-6fc75f7fc4-pvrxx_95013b21-1713-4225-9264-8b0ed08bd50d.kube-rbac-proxy-web.0.log {"@timestamp":"2025-05-14T19:26:44.183367204+00:00","message":"I0514 19:26:44.183191 1 log.go:194] http: TLS handshake error from 10.131.0.2:43056: write tcp 10.131.0.190:9091->10.131.0.2:43056: write: connection reset by peer","docker":{"container_id":"8c25cb84d4b32704450de7e3591160c25535eb5d455eed05dcd6ec20eff51d22"},"kubernetes":{"container_name":"kube-rbac-proxy-web","namespace_name":"openshift-monitoring","pod_name":"thanos-querier-6fc75f7fc4-pvrxx","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d049bad16dfe349671eef192f3e2dc0eaf27f69db6abfe054d434564c6ab0b15","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d049bad16dfe349671eef192f3e2dc0eaf27f69db6abfe054d434564c6ab0b15","pod_id":"95013b21-1713-4225-9264-8b0ed08bd50d","pod_ip":"10.131.0.190","host":"worker-0","labels":{"app_kubernetes_io_component":"query-layer","app_kubernetes_io_instance":"thanos-querier","app_kubernetes_io_managed-by":"cluster-monitoring-operator","app_kubernetes_io_name":"thanos-query","app_kubernetes_io_part-of":"openshift-monitoring","app_kubernetes_io_version":"0.32.5","pod-template-hash":"6fc75f7fc4"},"master_url":"https://kubernetes.default.svc","namespace_id":"29262c15-e202-4ba7-ac6e-3b833726c6fd","namespace_labels":{"kubernetes_io_metadata_name":"openshift-monitoring","name":"openshift-monitoring","network_openshift_io_policy-group":"monitoring","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["app_kubernetes_io_component=query-layer","app_kubernetes_io_instance=thanos-querier","app_kubernetes_io_managed-by=cluster-monitoring-operator","app_kubernetes_io_name=thanos-query","app_kubernetes_io_part-of=openshift-monitoring","app_kubernetes_io_version=0_32_5","pod-template-hash=6fc75f7fc4"]},"level":"info","hostname":"worker-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T19:26:44.185447+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":3666,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"clusterId":"ocp47-log507-19","logs":"5.8.20_415_infra"}},"viaq_msg_id":"YmMxNzE4YjgtZDFmMS00YzU0LWI1ZDAtZTJiODZjY2NmYmVj","log_type":"infrastructure"} 2025-05-14T19:26:44+00:00 kubernetes.var.log.pods.openshift-monitoring_thanos-querier-6fc75f7fc4-pvrxx_95013b21-1713-4225-9264-8b0ed08bd50d.kube-rbac-proxy-web.0.log {"@timestamp":"2025-05-14T19:26:44.673442887+00:00","message":"I0514 19:26:44.673312 1 log.go:194] http: TLS handshake error from 10.128.2.2:51068: write tcp 10.131.0.190:9091->10.128.2.2:51068: write: connection reset by peer","docker":{"container_id":"8c25cb84d4b32704450de7e3591160c25535eb5d455eed05dcd6ec20eff51d22"},"kubernetes":{"container_name":"kube-rbac-proxy-web","namespace_name":"openshift-monitoring","pod_name":"thanos-querier-6fc75f7fc4-pvrxx","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d049bad16dfe349671eef192f3e2dc0eaf27f69db6abfe054d434564c6ab0b15","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d049bad16dfe349671eef192f3e2dc0eaf27f69db6abfe054d434564c6ab0b15","pod_id":"95013b21-1713-4225-9264-8b0ed08bd50d","pod_ip":"10.131.0.190","host":"worker-0","labels":{"app_kubernetes_io_component":"query-layer","app_kubernetes_io_instance":"thanos-querier","app_kubernetes_io_managed-by":"cluster-monitoring-operator","app_kubernetes_io_name":"thanos-query","app_kubernetes_io_part-of":"openshift-monitoring","app_kubernetes_io_version":"0.32.5","pod-template-hash":"6fc75f7fc4"},"master_url":"https://kubernetes.default.svc","namespace_id":"29262c15-e202-4ba7-ac6e-3b833726c6fd","namespace_labels":{"kubernetes_io_metadata_name":"openshift-monitoring","name":"openshift-monitoring","network_openshift_io_policy-group":"monitoring","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["app_kubernetes_io_component=query-layer","app_kubernetes_io_instance=thanos-querier","app_kubernetes_io_managed-by=cluster-monitoring-operator","app_kubernetes_io_name=thanos-query","app_kubernetes_io_part-of=openshift-monitoring","app_kubernetes_io_version=0_32_5","pod-template-hash=6fc75f7fc4"]},"level":"info","hostname":"worker-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.177.189","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-05-14T19:26:44.674687+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":3667,"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"clusterId":"ocp47-log507-19","logs":"5.8.20_415_infra"}},"viaq_msg_id":"ZjllM2M0ZGYtYjYxYy00OWZhLTg5ZGEtM2JiNTk0MzMzNDIw","log_type":"infrastructure"} ```