Cluster logging 6.0.8 on OCP 4.15 CSV and Pods: ``` [root@rdr-jeev-415log-bastion-0 ~]# oc get csv -n openshift-logging NAME DISPLAY VERSION REPLACES PHASE cluster-logging.v6.0.8 Red Hat OpenShift Logging 6.0.8 cluster-logging.v6.0.7 Succeeded loki-operator.v6.0.8 Loki Operator 6.0.8 loki-operator.v6.0.7 Succeeded [root@rdr-jeev-415log-bastion-0 ~]# oc get pods -n openshift-logging NAME READY STATUS RESTARTS AGE cluster-logging-operator-668846c54c-6477k 1/1 Running 0 115m lokistack-ltetz-compactor-0 1/1 Running 0 113m lokistack-ltetz-distributor-7f55b6dfdc-pz9pd 1/1 Running 0 113m lokistack-ltetz-gateway-77979b95f7-gnb74 2/2 Running 0 113m lokistack-ltetz-gateway-77979b95f7-lnxln 2/2 Running 0 113m lokistack-ltetz-index-gateway-0 1/1 Running 0 113m lokistack-ltetz-ingester-0 1/1 Running 0 113m lokistack-ltetz-querier-6bcfb55494-jjn7c 1/1 Running 0 113m lokistack-ltetz-query-frontend-7fb4bdcb58-pmj5t 1/1 Running 0 113m lokistack-ltetz-ruler-0 1/1 Running 0 113m [root@rdr-jeev-415log-bastion-0 ~]# ``` Cluster logging: CloudWatch: Application logs: ``` {'timestamp': 1746546292838, 'message': '{"@timestamp":"2025-05-05T06:47:26.136418776Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.73/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:49\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"169.254.169.5/32\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.73/23\\",\\"gateway_ip\\":\\"10.131.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.73\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:49\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://f8117bb4e8064f5aafbec9052b54a317ebb6437cd1a6b1e74c72f8405733181f","container_image":"quay.io/powercloud/acmeair-authservice-java:1.0","container_image_id":"quay.io/powercloud/acmeair-authservice-java@sha256:ac57adc499c7d71ba889c90452b0786a7d96de3f83eb5d0db51bc9f373c3cad3","container_name":"acmeair-authservice-java","labels":{"name":"acmeair-auth-deployment","pod-template-hash":"6678875bd5"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"3bfbeacd-40a2-4d8e-9035-965ec21ac0bf","pod_ip":"10.131.0.73","pod_name":"acmeair-authservice-6678875bd5-f2d2j","pod_owner":"ReplicaSet/acmeair-authservice-6678875bd5"},"level":"default","log_source":"container","log_type":"application","message":"[AUDIT ] CWWKF0011I: The defaultServer server is ready to run a smarter planet. The defaultServer server started in 11.449 seconds.","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-application"},"sequence":1746546292578303233},"stream_name":"acme-air_acmeair-authservice-6678875bd5-f2d2j_acmeair-authservice-java"}', 'ingestionTime': 1746546301674} {'timestamp': 1746546292835, 'message': '{"@timestamp":"2025-05-05T06:49:27.744036057Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.74/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:4a\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"169.254.169.5/32\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.74/23\\",\\"gateway_ip\\":\\"10.131.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.74\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:4a\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://c15867d7ce33809e2792223af370b95e811fe5089032d6887a80ab73c4045f1b","container_image":"ibmcom/icp-mongodb-ppc64le:4.0.12","container_image_id":"docker.io/ibmcom/icp-mongodb-ppc64le@sha256:f54976512a396366c7947073c656e4bb2b33bd3177ae758a23e3630f37bec7d5","container_name":"acmeair-booking-db","labels":{"name":"acmeair-booking-db","pod-template-hash":"5ff54549bb"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"9fb014dc-f1f4-4b6e-ac0a-65d35dc26fb9","pod_ip":"10.131.0.74","pod_name":"acmeair-booking-db-5ff54549bb-vh9vg","pod_owner":"ReplicaSet/acmeair-booking-db-5ff54549bb"},"level":"default","log_source":"container","log_type":"application","message":"2025-05-05T06:49:27.743+0000 I NETWORK [conn3] received client metadata from 10.131.0.69:49722 conn3: { driver: { name: \\"mongo-java-driver|legacy\\", version: \\"3.10.1\\" }, os: { type: \\"Linux\\", name: \\"Linux\\", architecture: \\"ppc64le\\", version: \\"5.14.0-284.115.1.el9_2.ppc64le\\" }, platform: \\"Java/Eclipse OpenJ9/1.8.0_262-b10\\" }","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-application"},"sequence":1746546292575422895},"stream_name":"acme-air_acmeair-booking-db-5ff54549bb-vh9vg_acmeair-booking-db"}', 'ingestionTime': 1746546300582} {'timestamp': 1746546292831, 'message': '{"@timestamp":"2025-05-05T06:47:38.165789611Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.69/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:45\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"169.254.169.5/32\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.69/23\\",\\"gateway_ip\\":\\"10.131.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.69\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:45\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://7b63c3e6fea2bf688c44d8b0968cf61f121ab1c08dd67d84835492fd48ba1933","container_image":"quay.io/powercloud/acmeair-bookingservice-java:1.0","container_image_id":"quay.io/powercloud/acmeair-bookingservice-java@sha256:03c40c5bf1bbc60913651d5113c8d57ad436bb5031667389835bf6b67a8fbb83","container_name":"acmeair-bookingservice-java","labels":{"name":"acmeair-booking-deployment","pod-template-hash":"686b7cb778"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"6d1c42b4-2441-4a8f-b94d-70e994c5f4ed","pod_ip":"10.131.0.69","pod_name":"acmeair-bookingservice-686b7cb778-t8ttx","pod_owner":"ReplicaSet/acmeair-bookingservice-686b7cb778"},"level":"warn","log_source":"container","log_type":"application","message":"[WARNING ] No suitable sender found. Using NoopSender, meaning that data will not be sent anywhere!","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-application"},"sequence":1746546292571207723},"stream_name":"acme-air_acmeair-bookingservice-686b7cb778-t8ttx_acmeair-bookingservice-java"}', 'ingestionTime': 1746546299440} {'timestamp': 1746546292834, 'message': '{"@timestamp":"2025-05-05T06:49:27.625914323Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.72/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:48\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"169.254.169.5/32\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.72/23\\",\\"gateway_ip\\":\\"10.131.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.72\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:48\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://50c00a68858253a2f37aa74a33b647cc1973e036f42830e04177dc3418f76d9d","container_image":"ibmcom/icp-mongodb-ppc64le:4.0.12","container_image_id":"docker.io/ibmcom/icp-mongodb-ppc64le@sha256:f54976512a396366c7947073c656e4bb2b33bd3177ae758a23e3630f37bec7d5","container_name":"acmeair-customer-db","labels":{"name":"acmeair-customer-db","pod-template-hash":"cd47cd9fb"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"26d9e052-3412-4441-b724-6cacef73d826","pod_ip":"10.131.0.72","pod_name":"acmeair-customer-db-cd47cd9fb-6ms24","pod_owner":"ReplicaSet/acmeair-customer-db-cd47cd9fb"},"level":"default","log_source":"container","log_type":"application","message":"2025-05-05T06:49:27.625+0000 I NETWORK [conn3] received client metadata from 10.131.0.70:46930 conn3: { driver: { name: \\"mongo-java-driver|legacy\\", version: \\"3.10.1\\" }, os: { type: \\"Linux\\", name: \\"Linux\\", architecture: \\"ppc64le\\", version: \\"5.14.0-284.115.1.el9_2.ppc64le\\" }, platform: \\"Java/Eclipse OpenJ9/1.8.0_262-b10\\" }","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-application"},"sequence":1746546292574154093},"stream_name":"acme-air_acmeair-customer-db-cd47cd9fb-6ms24_acmeair-customer-db"}', 'ingestionTime': 1746546300320} ``` Audit logs: ``` {'timestamp': 1746546382799, 'message': '{"@timestamp":"2025-05-06T15:46:17.065+00:00","audit.linux":{"record_id":"51137","type":"PROCTITLE"},"hostname":"master-0","level":"default","log_source":"auditd","log_type":"audit","message":"type=PROCTITLE msg=audit(1746546377.065:51137): proctitle=2F7573722F62696E2F72756E63002D2D73797374656D642D6367726F7570002D2D726F6F743D2F72756E2F72756E6300637265617465002D2D62756E646C65002F72756E2F636F6E7461696E6572732F73746F726167652F6F7665726C61792D636F6E7461696E6572732F663630616361306262353036303735613930323061","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546382792103478},"stream_name":"master-0.auditd"}', 'ingestionTime': 1746546388285} {'timestamp': 1746546673173, 'message': '{"@timestamp":"2025-05-06T15:51:08.465705274Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \\"system:openshift:openshift-apiserver\\" of ClusterRole \\"cluster-admin\\" to ServiceAccount \\"openshift-apiserver-sa/openshift-apiserver\\""},"apiVersion":"audit.k8s.io/v1","auditID":"e1943a31-73c2-4aa6-8367-4be736dd7204","hostname":"master-0","k8s_audit_level":"Metadata","kind":"Event","level":"Metadata","log_source":"kubeAPI","log_type":"audit","objectRef":{"apiGroup":"authorization.k8s.io","apiVersion":"v1","resource":"subjectaccessreviews"},"openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546668466297341},"requestReceivedTimestamp":"2025-05-06T15:51:08.457997Z","requestURI":"/apis/authorization.k8s.io/v1/subjectaccessreviews?timeout=10s","responseStatus":{"code":201,"metadata":{}},"sourceIPs":["10.128.1.250"],"stage":"ResponseComplete","stageTimestamp":"2025-05-06T15:51:08.461362Z","stream_name":"master-0.kubeapi","user":{"extra":{"authentication.kubernetes.io/pod-name":["apiserver-6bd8bfc7df-gt4zk"],"authentication.kubernetes.io/pod-uid":["c9441d76-dec3-48e0-af95-62880c691c5b"]},"groups":["system:serviceaccounts","system:serviceaccounts:openshift-apiserver","system:authenticated"],"uid":"ec49b816-ebc9-4fed-8745-be7929678376","username":"system:serviceaccount:openshift-apiserver:openshift-apiserver-sa"},"userAgent":"openshift-apiserver/v0.0.0 (linux/ppc64le) kubernetes/$Format","verb":"create"}', 'ingestionTime': 1746546676725} {'timestamp': 1746546670678, 'message': '{"@timestamp":"2025-05-06T15:51:04.788806576Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \\"prometheus-k8s\\" of ClusterRole \\"prometheus-k8s\\" to ServiceAccount \\"prometheus-k8s/openshift-monitoring\\""},"apiVersion":"audit.k8s.io/v1","auditID":"5443c51c-f139-461e-ad26-1119361a0b7b","hostname":"master-0","kind":"Event","level":"Metadata","log_source":"openshiftAPI","log_type":"audit","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546664789269747},"openshift_audit_level":"Metadata","requestReceivedTimestamp":"2025-05-06T15:51:04.484650Z","requestURI":"/metrics","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.128.3.115"],"stage":"ResponseComplete","stageTimestamp":"2025-05-06T15:51:04.541292Z","stream_name":"master-0.openshiftapi","user":{"groups":["system:authenticated"],"username":"system:serviceaccount:openshift-monitoring:prometheus-k8s"},"userAgent":"Prometheus/2.48.0","verb":"get"}', 'ingestionTime': 1746546672807} {'timestamp': 1746546388407, 'message': '{"@timestamp":"2025-05-05T04:59:21.107+00:00","audit.linux":{"record_id":"14832","type":"BPF"},"hostname":"master-1","level":"default","log_source":"auditd","log_type":"audit","message":"type=BPF msg=audit(1746421161.107:14832): prog-id=0 op=UNLOAD","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546382264492341},"stream_name":"master-1.auditd"}', 'ingestionTime': 1746546390690} ``` Infrastructure logs: ``` {'timestamp': 1746546673150, 'message': '{"@timestamp":"2025-05-05T20:43:06.796654Z","_RUNTIME_SCOPE":"system","_STREAM_ID":"33d30a6adb414167a250ced0ac50fcd4","_SYSTEMD_INVOCATION_ID":"70afad6f335a4d02b220fbfba58df001","hostname":"master-0","level":"info","log_source":"node","log_type":"infrastructure","message":"I0505 20:43:06.796351 2533 kubelet_getters.go:187] \\"Pod status updated\\" pod=\\"openshift-kube-apiserver/kube-apiserver-master-0\\" status=\\"Running\\"","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-infrastructure"},"sequence":1746546663439155433},"stream_name":"master-0.journal.system","systemd":{"t":{"BOOT_ID":"4586239920664feb9666ba4c8e16d66c","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/bin/kubelet --config=/etc/kubernetes/kubelet.conf --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig --container-runtime-endpoint=/var/run/crio/crio.sock --runtime-cgroups=/system.slice/crio.service --node-labels=node-role.kubernetes.io/control-plane,node-role.kubernetes.io/master,node.openshift.io/os_id=rhcos --node-ip=10.20.177.64 --minimum-container-ttl-duration=6m0s --cloud-provider= --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec --hostname-override= --provider-id= --register-with-taints=node-role.kubernetes.io/master=:NoSchedule --pod-infra-container-image=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:546b3635ab9f651032bf142e243809d64391ee7fcdec9c3d0636218c3dd7aa08 --system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=1Gi --v=2","COMM":"kubelet","EXE":"/usr/bin/kubelet","GID":"0","MACHINE_ID":"391a7bcf45e544029be895a9654ed7ee","PID":"2533","SELINUX_CONTEXT":"system_u:system_r:kubelet_t:s0-s0:c0.c1023","STREAM_ID":"33d30a6adb414167a250ced0ac50fcd4","SYSTEMD_CGROUP":"/system.slice/kubelet.service","SYSTEMD_INVOCATION_ID":"70afad6f335a4d02b220fbfba58df001","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"kubelet.service","TRANSPORT":"stdout","UID":"0"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"kubenswrapper"}},"time":"2025-05-05T20:43:06+00:00"}', 'ingestionTime': 1746546676483} {'timestamp': 1746546632203, 'message': '{"@timestamp":"2025-05-06T15:38:34.520721677Z","hostname":"master-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.128.1.250/23\\"],\\"mac_address\\":\\"0a:58:0a:80:01:fa\\",\\"gateway_ips\\":[\\"10.128.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"169.254.169.5/32\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.128.0.1\\"}],\\"ip_address\\":\\"10.128.1.250/23\\",\\"gateway_ip\\":\\"10.128.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.128.1.250\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:80:01:fa\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/required-scc":"privileged","openshift.io/scc":"privileged","operator.openshift.io/dep-desired.generation":"6","operator.openshift.io/dep-openshift-apiserver.config.configmap":"2Jsrrw==","operator.openshift.io/dep-openshift-apiserver.etcd-client.secret":"3Poe4g==","operator.openshift.io/dep-openshift-apiserver.etcd-serving-ca.configmap":"41537Q==","operator.openshift.io/dep-openshift-apiserver.image-import-ca.configmap":"pohL8g==","operator.openshift.io/dep-openshift-apiserver.trusted-ca-bundle.configmap":"ElMHxA=="},"container_id":"cri-o://19e4c5e136e6b2c235fd543ca067299bd8a3e3b4a468115a793a1c1b93e42f84","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf3e4ed10a434395e26a0df443e8c41ec4db1cc64b4e4d291639da5db0a7ae95","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf3e4ed10a434395e26a0df443e8c41ec4db1cc64b4e4d291639da5db0a7ae95","container_name":"openshift-apiserver","labels":{"apiserver":"true","app":"openshift-apiserver-a","openshift-apiserver-anti-affinity":"true","pod-template-hash":"6bd8bfc7df","revision":"1"},"namespace_id":"73810a40-c12b-492f-abd7-cd16f20f57f0","namespace_labels":{"kubernetes_io_metadata_name":"openshift-apiserver","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-apiserver","pod_id":"c9441d76-dec3-48e0-af95-62880c691c5b","pod_ip":"10.128.1.250","pod_name":"apiserver-6bd8bfc7df-gt4zk","pod_owner":"ReplicaSet/apiserver-6bd8bfc7df"},"level":"default","log_source":"container","log_type":"infrastructure","message":"}. Err: connection error: desc = \\"error reading server preface: read tcp 10.128.1.250:44964->10.20.177.77:2379: use of closed network connection\\"","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-infrastructure"},"sequence":1746546628395590066},"stream_name":"master-0.openshift-apiserver_apiserver-6bd8bfc7df-gt4zk_openshift-apiserver"}', 'ingestionTime': 1746546638113} {'timestamp': 1746546632208, 'message': '{"@timestamp":"2025-05-05T06:47:43.625852607Z","hostname":"master-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.128.1.250/23\\"],\\"mac_address\\":\\"0a:58:0a:80:01:fa\\",\\"gateway_ips\\":[\\"10.128.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"169.254.169.5/32\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.128.0.1\\"}],\\"ip_address\\":\\"10.128.1.250/23\\",\\"gateway_ip\\":\\"10.128.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.128.1.250\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:80:01:fa\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/required-scc":"privileged","openshift.io/scc":"privileged","operator.openshift.io/dep-desired.generation":"6","operator.openshift.io/dep-openshift-apiserver.config.configmap":"2Jsrrw==","operator.openshift.io/dep-openshift-apiserver.etcd-client.secret":"3Poe4g==","operator.openshift.io/dep-openshift-apiserver.etcd-serving-ca.configmap":"41537Q==","operator.openshift.io/dep-openshift-apiserver.image-import-ca.configmap":"pohL8g==","operator.openshift.io/dep-openshift-apiserver.trusted-ca-bundle.configmap":"ElMHxA=="},"container_id":"cri-o://a0503f5487379d94fc5201867b955e651ee0e8a5ac359b7e850d3b4beb84dd9a","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:483a0f3d46f7d551ccf43cd0458d4cfab30486317820750d7a36fd2e185f2353","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:483a0f3d46f7d551ccf43cd0458d4cfab30486317820750d7a36fd2e185f2353","container_name":"openshift-apiserver-check-endpoints","labels":{"apiserver":"true","app":"openshift-apiserver-a","openshift-apiserver-anti-affinity":"true","pod-template-hash":"6bd8bfc7df","revision":"1"},"namespace_id":"73810a40-c12b-492f-abd7-cd16f20f57f0","namespace_labels":{"kubernetes_io_metadata_name":"openshift-apiserver","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-apiserver","pod_id":"c9441d76-dec3-48e0-af95-62880c691c5b","pod_ip":"10.128.1.250","pod_name":"apiserver-6bd8bfc7df-gt4zk","pod_owner":"ReplicaSet/apiserver-6bd8bfc7df"},"level":"info","log_source":"container","log_type":"infrastructure","message":"I0505 06:47:43.625826 1 base_controller.go:110] Starting #1 worker of check-endpoints controller ...","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-infrastructure"},"sequence":1746546628398170016},"stream_name":"master-0.openshift-apiserver_apiserver-6bd8bfc7df-gt4zk_openshift-apiserver-check-endpoints"}', 'ingestionTime': 1746546636068} {'timestamp': 1746546512452, 'message': '{"@timestamp":"2025-05-06T06:20:33.838613050Z","hostname":"master-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.128.1.252/23\\"],\\"mac_address\\":\\"0a:58:0a:80:01:fc\\",\\"gateway_ips\\":[\\"10.128.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"169.254.169.5/32\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.128.0.1\\"}],\\"ip_address\\":\\"10.128.1.252/23\\",\\"gateway_ip\\":\\"10.128.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.128.1.252\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:80:01:fc\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/required-scc":"privileged","openshift.io/scc":"privileged","operator.openshift.io/bootstrap-user-exists":"true","operator.openshift.io/rvs-hash":"hQN0bCkv7GQ_cPWiyAL5D4RwE3c7-gyz9EGCOpwpodn1bX13n0fZzSC1sQfxaexAfIABknRh9Y34ucZle0dQxg"},"container_id":"cri-o://dc4b252280a30d0ada82e6c227c4ae24c5cd9f706a0c02a76943ecdf28984b44","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e3b7ccffc33ee45b34973df72b57cbc625543c159dd3d9e165aada85d9bb3eb","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e3b7ccffc33ee45b34973df72b57cbc625543c159dd3d9e165aada85d9bb3eb","container_name":"oauth-openshift","labels":{"app":"oauth-openshift","oauth-openshift-anti-affinity":"true","pod-template-hash":"556d7c464"},"namespace_id":"d608a2f6-4be3-4bf5-a8d3-9d5a9f1645e5","namespace_labels":{"kubernetes_io_metadata_name":"openshift-authentication","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-authentication","pod_id":"3357356f-a659-4d2f-8ea9-65c057139fb4","pod_ip":"10.128.1.252","pod_name":"oauth-openshift-556d7c464-2lm2h","pod_owner":"ReplicaSet/oauth-openshift-556d7c464"},"level":"warn","log_source":"container","log_type":"infrastructure","message":"W0506 06:20:33.838529 1 context.go:119] Failed to set annotations[\\"authentication.openshift.io/username\\"] to \\"kube:admin\\" for audit:\\"02e2d129-b7fd-42ee-9dc6-7652410c0092\\", it has already been set to \\"kubeadmin\\"","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-infrastructure"},"sequence":1746546508270627289},"stream_name":"master-0.openshift-authentication_oauth-openshift-556d7c464-2lm2h_oauth-openshift"}', 'ingestionTime': 1746546516091} ``` Elasticsearch : Application logs: ``` {"_index":"application","_type":"_doc","_id":"dmFFppYBjyXErPejfB5R","_score":8.69915E-6,"_source":{"@timestamp":"2025-05-05T06:47:17.632685328Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.69/23\"],\"mac_address\":\"0a:58:0a:83:00:45\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.69/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.69\"\n ],\n \"mac\": \"0a:58:0a:83:00:45\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://7b63c3e6fea2bf688c44d8b0968cf61f121ab1c08dd67d84835492fd48ba1933","container_image":"quay.io/powercloud/acmeair-bookingservice-java:1.0","container_image_id":"quay.io/powercloud/acmeair-bookingservice-java@sha256:03c40c5bf1bbc60913651d5113c8d57ad436bb5031667389835bf6b67a8fbb83","container_name":"acmeair-bookingservice-java","labels":{"name":"acmeair-booking-deployment","pod-template-hash":"686b7cb778"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"6d1c42b4-2441-4a8f-b94d-70e994c5f4ed","pod_ip":"10.131.0.69","pod_name":"acmeair-bookingservice-686b7cb778-t8ttx","pod_owner":"ReplicaSet/acmeair-bookingservice-686b7cb778"},"level":"default","log_source":"container","log_type":"application","message":"Launching defaultServer (Open Liberty 20.0.0.9/wlp-1.0.44.cl200920200820-0913) on Eclipse OpenJ9 VM, version 1.8.0_262-b10 (en_US)","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-application"},"sequence":1746546292570555775}}},{"_index":"application","_type":"_doc","_id":"d2FFppYBjyXErPejfB5R","_score":8.69915E-6,"_source":{"@timestamp":"2025-05-05T06:47:17.824285645Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.69/23\"],\"mac_address\":\"0a:58:0a:83:00:45\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.69/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.69\"\n ],\n \"mac\": \"0a:58:0a:83:00:45\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://7b63c3e6fea2bf688c44d8b0968cf61f121ab1c08dd67d84835492fd48ba1933","container_image":"quay.io/powercloud/acmeair-bookingservice-java:1.0","container_image_id":"quay.io/powercloud/acmeair-bookingservice-java@sha256:03c40c5bf1bbc60913651d5113c8d57ad436bb5031667389835bf6b67a8fbb83","container_name":"acmeair-bookingservice-java","labels":{"name":"acmeair-booking-deployment","pod-template-hash":"686b7cb778"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"6d1c42b4-2441-4a8f-b94d-70e994c5f4ed","pod_ip":"10.131.0.69","pod_name":"acmeair-bookingservice-686b7cb778-t8ttx","pod_owner":"ReplicaSet/acmeair-bookingservice-686b7cb778"},"level":"default","log_source":"container","log_type":"application","message":"[AUDIT ] CWWKE0001I: The server defaultServer has been launched.","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-application"},"sequence":1746546292570601168}}},{"_index":"application","_type":"_doc","_id":"eGFFppYBjyXErPejfB5R","_score":8.69915E-6,"_source":{"@timestamp":"2025-05-05T06:47:20.453971457Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.69/23\"],\"mac_address\":\"0a:58:0a:83:00:45\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.69/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.69\"\n ],\n \"mac\": \"0a:58:0a:83:00:45\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://7b63c3e6fea2bf688c44d8b0968cf61f121ab1c08dd67d84835492fd48ba1933","container_image":"quay.io/powercloud/acmeair-bookingservice-java:1.0","container_image_id":"quay.io/powercloud/acmeair-bookingservice-java@sha256:03c40c5bf1bbc60913651d5113c8d57ad436bb5031667389835bf6b67a8fbb83","container_name":"acmeair-bookingservice-java","labels":{"name":"acmeair-booking-deployment","pod-template-hash":"686b7cb778"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"6d1c42b4-2441-4a8f-b94d-70e994c5f4ed","pod_ip":"10.131.0.69","pod_name":"acmeair-bookingservice-686b7cb778-t8ttx","pod_owner":"ReplicaSet/acmeair-bookingservice-686b7cb778"},"level":"default","log_source":"container","log_type":"application","message":"[AUDIT ] CWWKG0093A: Processing configuration drop-ins resource: /opt/ol/wlp/usr/servers/defaultServer/configDropins/defaults/keystore.xml","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-application"},"sequence":1746546292570639060}}},{"_index":"application","_type":"_doc","_id":"eWFFppYBjyXErPejfB5R","_score":8.69915E-6,"_source":{"@timestamp":"2025-05-05T06:47:20.483557009Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.69/23\"],\"mac_address\":\"0a:58:0a:83:00:45\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.69/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.69\"\n ],\n \"mac\": \"0a:58:0a:83:00:45\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://7b63c3e6fea2bf688c44d8b0968cf61f121ab1c08dd67d84835492fd48ba1933","container_image":"quay.io/powercloud/acmeair-bookingservice-java:1.0","container_image_id":"quay.io/powercloud/acmeair-bookingservice-java@sha256:03c40c5bf1bbc60913651d5113c8d57ad436bb5031667389835bf6b67a8fbb83","container_name":"acmeair-bookingservice-java","labels":{"name":"acmeair-booking-deployment","pod-template-hash":"686b7cb778"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"6d1c42b4-2441-4a8f-b94d-70e994c5f4ed","pod_ip":"10.131.0.69","pod_name":"acmeair-bookingservice-686b7cb778-t8ttx","pod_owner":"ReplicaSet/acmeair-bookingservice-686b7cb778"},"level":"default","log_source":"container","log_type":"application","message":"[AUDIT ] CWWKG0093A: Processing configuration drop-ins resource: /opt/ol/wlp/usr/servers/defaultServer/configDropins/defaults/open-default-port.xml","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-application"},"sequence":1746546292570674963}}},{"_index":"application","_type":"_doc","_id":"emFFppYBjyXErPejfB5R","_score":8.69915E-6,"_source":{"@timestamp":"2025-05-05T06:47:21.687755673Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.69/23\"],\"mac_address\":\"0a:58:0a:83:00:45\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.69/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.69\"\n ],\n \"mac\": \"0a:58:0a:83:00:45\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://7b63c3e6fea2bf688c44d8b0968cf61f121ab1c08dd67d84835492fd48ba1933","container_image":"quay.io/powercloud/acmeair-bookingservice-java:1.0","container_image_id":"quay.io/powercloud/acmeair-bookingservice-java@sha256:03c40c5bf1bbc60913651d5113c8d57ad436bb5031667389835bf6b67a8fbb83","container_name":"acmeair-bookingservice-java","labels":{"name":"acmeair-booking-deployment","pod-template-hash":"686b7cb778"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"6d1c42b4-2441-4a8f-b94d-70e994c5f4ed","pod_ip":"10.131.0.69","pod_name":"acmeair-bookingservice-686b7cb778-t8ttx","pod_owner":"ReplicaSet/acmeair-bookingservice-686b7cb778"},"level":"default","log_source":"container","log_type":"application","message":"[AUDIT ] CWWKZ0058I: Monitoring dropins for applications.","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-application"},"sequence":1746546292570706883}}},{"_index":"application","_type":"_doc","_id":"e2FFppYBjyXErPejfB5R","_score":8.69915E-6,"_source":{"@timestamp":"2025-05-05T06:47:22.822090675Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.69/23\"],\"mac_address\":\"0a:58:0a:83:00:45\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.69/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.69\"\n ],\n \"mac\": \"0a:58:0a:83:00:45\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://7b63c3e6fea2bf688c44d8b0968cf61f121ab1c08dd67d84835492fd48ba1933","container_image":"quay.io/powercloud/acmeair-bookingservice-java:1.0","container_image_id":"quay.io/powercloud/acmeair-bookingservice-java@sha256:03c40c5bf1bbc60913651d5113c8d57ad436bb5031667389835bf6b67a8fbb83","container_name":"acmeair-bookingservice-java","labels":{"name":"acmeair-booking-deployment","pod-template-hash":"686b7cb778"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"6d1c42b4-2441-4a8f-b94d-70e994c5f4ed","pod_ip":"10.131.0.69","pod_name":"acmeair-bookingservice-686b7cb778-t8ttx","pod_owner":"ReplicaSet/acmeair-bookingservice-686b7cb778"},"level":"default","log_source":"container","log_type":"application","message":"Failed to find runtime version","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-application"},"sequence":1746546292570735722}}},{"_index":"application","_type":"_doc","_id":"fGFFppYBjyXErPejfB5R","_score":8.69915E-6,"_source":{"@timestamp":"2025-05-05T06:47:24.711663514Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.69/23\"],\"mac_address\":\"0a:58:0a:83:00:45\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.69/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.69\"\n ],\n \"mac\": \"0a:58:0a:83:00:45\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://7b63c3e6fea2bf688c44d8b0968cf61f121ab1c08dd67d84835492fd48ba1933","container_image":"quay.io/powercloud/acmeair-bookingservice-java:1.0","container_image_id":"quay.io/powercloud/acmeair-bookingservice-java@sha256:03c40c5bf1bbc60913651d5113c8d57ad436bb5031667389835bf6b67a8fbb83","container_name":"acmeair-bookingservice-java","labels":{"name":"acmeair-booking-deployment","pod-template-hash":"686b7cb778"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"6d1c42b4-2441-4a8f-b94d-70e994c5f4ed","pod_ip":"10.131.0.69","pod_name":"acmeair-bookingservice-686b7cb778-t8ttx","pod_owner":"ReplicaSet/acmeair-bookingservice-686b7cb778"},"level":"default","log_source":"container","log_type":"application","message":"[AUDIT ] CWWKT0016I: Web application available (default_host): http://acmeair-bookingservice-686b7cb778-t8ttx:9080/jwt/","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-application"},"sequence":1746546292570772287}}},{"_index":"application","_type":"_doc","_id":"fWFFppYBjyXErPejfB5R","_score":8.69915E-6,"_source":{"@timestamp":"2025-05-05T06:47:24.712266640Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.69/23\"],\"mac_address\":\"0a:58:0a:83:00:45\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.69/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.69\"\n ],\n \"mac\": \"0a:58:0a:83:00:45\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://7b63c3e6fea2bf688c44d8b0968cf61f121ab1c08dd67d84835492fd48ba1933","container_image":"quay.io/powercloud/acmeair-bookingservice-java:1.0","container_image_id":"quay.io/powercloud/acmeair-bookingservice-java@sha256:03c40c5bf1bbc60913651d5113c8d57ad436bb5031667389835bf6b67a8fbb83","container_name":"acmeair-bookingservice-java","labels":{"name":"acmeair-booking-deployment","pod-template-hash":"686b7cb778"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"6d1c42b4-2441-4a8f-b94d-70e994c5f4ed","pod_ip":"10.131.0.69","pod_name":"acmeair-bookingservice-686b7cb778-t8ttx","pod_owner":"ReplicaSet/acmeair-bookingservice-686b7cb778"},"level":"default","log_source":"container","log_type":"application","message":"[AUDIT ] CWWKT0016I: Web application available (default_host): http://acmeair-bookingservice-686b7cb778-t8ttx:9080/openapi/","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-application"},"sequence":1746546292570805965}}},{"_index":"application","_type":"_doc","_id":"fmFFppYBjyXErPejfB5R","_score":8.69915E-6,"_source":{"@timestamp":"2025-05-05T06:47:24.712995609Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.69/23\"],\"mac_address\":\"0a:58:0a:83:00:45\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.69/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.69\"\n ],\n \"mac\": \"0a:58:0a:83:00:45\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://7b63c3e6fea2bf688c44d8b0968cf61f121ab1c08dd67d84835492fd48ba1933","container_image":"quay.io/powercloud/acmeair-bookingservice-java:1.0","container_image_id":"quay.io/powercloud/acmeair-bookingservice-java@sha256:03c40c5bf1bbc60913651d5113c8d57ad436bb5031667389835bf6b67a8fbb83","container_name":"acmeair-bookingservice-java","labels":{"name":"acmeair-booking-deployment","pod-template-hash":"686b7cb778"},"namespace_id":"887394ee-9d1f-4888-8769-8e490f4a9b06","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"6d1c42b4-2441-4a8f-b94d-70e994c5f4ed","pod_ip":"10.131.0.69","pod_name":"acmeair-bookingservice-686b7cb778-t8ttx","pod_owner":"ReplicaSet/acmeair-bookingservice-686b7cb778"},"level":"default","log_source":"container","log_type":"application","message":"[AUDIT ] CWWKT0016I: Web application available (default_host): http://acmeair-bookingservice-686b7cb778-t8ttx:9080/ibm/api/","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-application"},"sequence":1746546292570841195}}}]}} ``` Audit logs: ``` {"took":3651,"timed_out":false,"_shards":{"total":2,"successful":2,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":1.586461E-6,"hits":[{"_index":"audit","_type":"_doc","_id":"V19FppYBjyXErPejM05c","_score":1.586461E-6,"_source":{"@timestamp":"2025-05-06T15:44:33.895172423Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:kube-controller-manager\" of ClusterRole \"system:kube-controller-manager\" to User \"system:kube-controller-manager\""},"apiVersion":"audit.k8s.io/v1","auditID":"683462a8-153e-4539-b38e-45c586345971","hostname":"master-1","k8s_audit_level":"Metadata","kind":"Event","level":"Metadata","log_source":"kubeAPI","log_type":"audit","objectRef":{"apiGroup":"coordination.k8s.io","apiVersion":"v1","name":"kube-controller-manager","namespace":"kube-system","resource":"leases"},"openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546274145864338},"requestReceivedTimestamp":"2025-05-06T14:19:00.907080Z","requestURI":"/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/kube-controller-manager?timeout=6s","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.20.177.13"],"stage":"ResponseComplete","stageTimestamp":"2025-05-06T14:19:00.909111Z","user":{"groups":["system:authenticated"],"username":"system:kube-controller-manager"},"userAgent":"kube-controller-manager/v1.28.15+f383677 (linux/ppc64le) kubernetes/4cf5291/leader-election","verb":"get"}},{"_index":"audit","_type":"_doc","_id":"3V9FppYBjyXErPejNF-z","_score":1.586461E-6,"_source":{"@timestamp":"2025-05-02T05:40:44.600+00:00","audit.linux":{"record_id":"12","type":"CONFIG_CHANGE"},"hostname":"master-2","level":"default","log_source":"auditd","log_type":"audit","message":"type=CONFIG_CHANGE msg=audit(1746164444.600:12): auid=4294967295 ses=4294967295 subj=system_u:system_r:unconfined_service_t:s0 op=add_rule key=(null) list=5 res=1\u001dAUID=\"unset\"","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546274269423043}}},{"_index":"audit","_type":"_doc","_id":"319FppYBjyXErPejNF-z","_score":1.586461E-6,"_source":{"@timestamp":"2025-05-02T05:40:44.600+00:00","audit.linux":{"record_id":"12","type":"PROCTITLE"},"hostname":"master-2","level":"default","log_source":"auditd","log_type":"audit","message":"type=PROCTITLE msg=audit(1746164444.600:12): proctitle=2F7362696E2F617564697463746C002D52002F6574632F61756469742F61756469742E72756C6573","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546274269607101}}},{"_index":"audit","_type":"_doc","_id":"4F9FppYBjyXErPejNF-z","_score":1.586461E-6,"_source":{"@timestamp":"2025-05-02T05:40:44.600+00:00","audit.linux":{"record_id":"13","type":"SERVICE_START"},"hostname":"master-2","level":"default","log_source":"auditd","log_type":"audit","message":"type=SERVICE_START msg=audit(1746164444.600:13): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=auditd comm=\"systemd\" exe=\"/usr/lib/systemd/systemd\" hostname=? addr=? terminal=? res=success'\u001dUID=\"root\" AUID=\"unset\"","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546274269702989}}},{"_index":"audit","_type":"_doc","_id":"4V9FppYBjyXErPejNF-z","_score":1.586461E-6,"_source":{"@timestamp":"2025-05-02T05:40:44.610+00:00","audit.linux":{"record_id":"14","type":"SYSTEM_BOOT"},"hostname":"master-2","level":"default","log_source":"auditd","log_type":"audit","message":"type=SYSTEM_BOOT msg=audit(1746164444.610:14): pid=1905 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg=' comm=\"systemd-update-utmp\" exe=\"/usr/lib/systemd/systemd-update-utmp\" hostname=? addr=? terminal=? res=success'\u001dUID=\"root\" AUID=\"unset\"","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546274269801184}}},{"_index":"audit","_type":"_doc","_id":"4l9FppYBjyXErPejNF-z","_score":1.586461E-6,"_source":{"@timestamp":"2025-05-02T05:40:44.610+00:00","audit.linux":{"record_id":"15","type":"SERVICE_START"},"hostname":"master-2","level":"default","log_source":"auditd","log_type":"audit","message":"type=SERVICE_START msg=audit(1746164444.610:15): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=systemd-update-utmp comm=\"systemd\" exe=\"/usr/lib/systemd/systemd\" hostname=? addr=? terminal=? res=success'\u001dUID=\"root\" AUID=\"unset\"","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546274269903135}}},{"_index":"audit","_type":"_doc","_id":"419FppYBjyXErPejNF-z","_score":1.586461E-6,"_source":{"@timestamp":"2025-05-02T05:40:44.630+00:00","audit.linux":{"record_id":"16","type":"BPF"},"hostname":"master-2","level":"default","log_source":"auditd","log_type":"audit","message":"type=BPF msg=audit(1746164444.630:16): prog-id=19 op=LOAD","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546274269991341}}},{"_index":"audit","_type":"_doc","_id":"W19FppYBjyXErPejM05c","_score":1.586461E-6,"_source":{"@timestamp":"2025-05-06T15:44:33.895259184Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":""},"apiVersion":"audit.k8s.io/v1","auditID":"6353ac69-5d47-4dd6-a04e-fa56e37352d8","hostname":"master-1","k8s_audit_level":"Metadata","kind":"Event","level":"Metadata","log_source":"kubeAPI","log_type":"audit","objectRef":{"apiGroup":"discovery.k8s.io","apiVersion":"v1","name":"kubernetes","namespace":"default","resource":"endpointslices"},"openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546274146267629},"requestReceivedTimestamp":"2025-05-06T14:19:01.247238Z","requestURI":"/apis/discovery.k8s.io/v1/namespaces/default/endpointslices/kubernetes","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["::1"],"stage":"ResponseComplete","stageTimestamp":"2025-05-06T14:19:01.248451Z","user":{"groups":["system:masters"],"uid":"9d5546a8-61c6-4651-846e-872aa4e3f9f6","username":"system:apiserver"},"userAgent":"kube-apiserver/v1.28.15+f383677 (linux/ppc64le) kubernetes/4cf5291","verb":"get"}},{"_index":"audit","_type":"_doc","_id":"5V9FppYBjyXErPejNF-z","_score":1.586461E-6,"_source":{"@timestamp":"2025-05-02T05:40:44.660+00:00","audit.linux":{"record_id":"18","type":"BPF"},"hostname":"master-2","level":"default","log_source":"auditd","log_type":"audit","message":"type=BPF msg=audit(1746164444.660:18): prog-id=20 op=LOAD","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546274270170034}}},{"_index":"audit","_type":"_doc","_id":"5l9FppYBjyXErPejNF-z","_score":1.586461E-6,"_source":{"@timestamp":"2025-05-02T05:40:44.660+00:00","audit.linux":{"record_id":"19","type":"BPF"},"hostname":"master-2","level":"default","log_source":"auditd","log_type":"audit","message":"type=BPF msg=audit(1746164444.660:19): prog-id=21 op=LOAD","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-audit"},"sequence":1746546274270256664}}}]}} ``` Infrastructure logs: ``` {"_index":"infrastructure","_type":"_doc","_id":"BF9FppYBjyXErPejM0Ra","_score":0.012046979,"_source":{"@timestamp":"2025-05-05T06:18:08.859535Z","INVOCATION_ID":"f6934be2d5224c14b2d04b24d0175334","TID":"1","_RUNTIME_SCOPE":"system","hostname":"master-1","level":"info","log_source":"node","log_type":"infrastructure","message":"crio-3cf0ceef556a20ac03ca339bdc88eff06f05921aafb6d8648e7ed20f8f6ff82a.scope: Deactivated successfully.","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-infrastructure"},"sequence":1746546273923756886},"systemd":{"t":{"BOOT_ID":"274588dcf3fa4b889eb852e04ce2da6b","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/lib/systemd/systemd --switched-root --system --deserialize 28","COMM":"systemd","EXE":"/usr/lib/systemd/systemd","GID":"0","MACHINE_ID":"42ce264bc9a44b0e887485fc1297c3d3","PID":"1","SELINUX_CONTEXT":"system_u:system_r:init_t:s0","SYSTEMD_CGROUP":"/init.scope","SYSTEMD_SLICE":"-.slice","SYSTEMD_UNIT":"init.scope","TRANSPORT":"journal","UID":"0"},"u":{"CODE_FILE":"src/core/unit.c","CODE_FUNCTION":"unit_log_success","CODE_LINE":"5593","MESSAGE_ID":"7ad2d189f7e94e70a38c781354912448","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"systemd","UNIT":"crio-3cf0ceef556a20ac03ca339bdc88eff06f05921aafb6d8648e7ed20f8f6ff82a.scope"}},"tag":".journal.system","time":"2025-05-05T06:18:08+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"BV9FppYBjyXErPejM0Ra","_score":0.012046979,"_source":{"@timestamp":"2025-05-05T06:18:08.860131Z","CPU_USAGE_NSEC":"9435306000","INVOCATION_ID":"f6934be2d5224c14b2d04b24d0175334","TID":"1","_RUNTIME_SCOPE":"system","hostname":"master-1","level":"info","log_source":"node","log_type":"infrastructure","message":"crio-3cf0ceef556a20ac03ca339bdc88eff06f05921aafb6d8648e7ed20f8f6ff82a.scope: Consumed 9.435s CPU time.","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-infrastructure"},"sequence":1746546273923916059},"systemd":{"t":{"BOOT_ID":"274588dcf3fa4b889eb852e04ce2da6b","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/lib/systemd/systemd --switched-root --system --deserialize 28","COMM":"systemd","EXE":"/usr/lib/systemd/systemd","GID":"0","MACHINE_ID":"42ce264bc9a44b0e887485fc1297c3d3","PID":"1","SELINUX_CONTEXT":"system_u:system_r:init_t:s0","SYSTEMD_CGROUP":"/init.scope","SYSTEMD_SLICE":"-.slice","SYSTEMD_UNIT":"init.scope","TRANSPORT":"journal","UID":"0"},"u":{"CODE_FILE":"src/core/unit.c","CODE_FUNCTION":"unit_log_resources","CODE_LINE":"2483","MESSAGE_ID":"ae8f7b866b0347b9af31fe1c80b127c0","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"systemd","UNIT":"crio-3cf0ceef556a20ac03ca339bdc88eff06f05921aafb6d8648e7ed20f8f6ff82a.scope"}},"tag":".journal.system","time":"2025-05-05T06:18:08+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"Bl9FppYBjyXErPejM0Ra","_score":0.012046979,"_source":{"@timestamp":"2025-05-05T06:18:08.913252Z","INVOCATION_ID":"66add534c21643609c66fc104fff9713","TID":"1","_RUNTIME_SCOPE":"system","hostname":"master-1","level":"info","log_source":"node","log_type":"infrastructure","message":"crio-conmon-3cf0ceef556a20ac03ca339bdc88eff06f05921aafb6d8648e7ed20f8f6ff82a.scope: Deactivated successfully.","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-infrastructure"},"sequence":1746546273924064755},"systemd":{"t":{"BOOT_ID":"274588dcf3fa4b889eb852e04ce2da6b","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/lib/systemd/systemd --switched-root --system --deserialize 28","COMM":"systemd","EXE":"/usr/lib/systemd/systemd","GID":"0","MACHINE_ID":"42ce264bc9a44b0e887485fc1297c3d3","PID":"1","SELINUX_CONTEXT":"system_u:system_r:init_t:s0","SYSTEMD_CGROUP":"/init.scope","SYSTEMD_SLICE":"-.slice","SYSTEMD_UNIT":"init.scope","TRANSPORT":"journal","UID":"0"},"u":{"CODE_FILE":"src/core/unit.c","CODE_FUNCTION":"unit_log_success","CODE_LINE":"5593","MESSAGE_ID":"7ad2d189f7e94e70a38c781354912448","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"systemd","UNIT":"crio-conmon-3cf0ceef556a20ac03ca339bdc88eff06f05921aafb6d8648e7ed20f8f6ff82a.scope"}},"tag":".journal.system","time":"2025-05-05T06:18:08+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"B19FppYBjyXErPejM0Ra","_score":0.012046979,"_source":{"@timestamp":"2025-05-05T06:18:09.010840Z","_RUNTIME_SCOPE":"system","_STREAM_ID":"0fa7e533314547b8a24544289b1279e2","_SYSTEMD_INVOCATION_ID":"11e1751345034868ab781f1b98f304ed","hostname":"master-1","level":"info","log_source":"node","log_type":"infrastructure","message":"time=\"2025-05-05 06:18:09.010791024Z\" level=info msg=\"Stopped container 3cf0ceef556a20ac03ca339bdc88eff06f05921aafb6d8648e7ed20f8f6ff82a: openshift-logging/collector-dczxq/collector\" id=16bf4e01-2c75-4367-8153-17085c0fe0f8 name=/runtime.v1.RuntimeService/StopContainer","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-infrastructure"},"sequence":1746546273924195063},"systemd":{"t":{"BOOT_ID":"274588dcf3fa4b889eb852e04ce2da6b","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/bin/crio","COMM":"crio","EXE":"/usr/bin/crio","GID":"0","MACHINE_ID":"42ce264bc9a44b0e887485fc1297c3d3","PID":"2481","SELINUX_CONTEXT":"system_u:system_r:container_runtime_t:s0","STREAM_ID":"0fa7e533314547b8a24544289b1279e2","SYSTEMD_CGROUP":"/system.slice/crio.service","SYSTEMD_INVOCATION_ID":"11e1751345034868ab781f1b98f304ed","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"crio.service","TRANSPORT":"stdout","UID":"0"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"crio"}},"tag":".journal.system","time":"2025-05-05T06:18:09+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"CF9FppYBjyXErPejM0Ra","_score":0.012046979,"_source":{"@timestamp":"2025-05-05T06:18:09.011411Z","_RUNTIME_SCOPE":"system","_STREAM_ID":"0fa7e533314547b8a24544289b1279e2","_SYSTEMD_INVOCATION_ID":"11e1751345034868ab781f1b98f304ed","hostname":"master-1","level":"info","log_source":"node","log_type":"infrastructure","message":"time=\"2025-05-05 06:18:09.011391210Z\" level=info msg=\"Stopping pod sandbox: 6eaaa1a0122eeed9f8ed23d8d75208891f9349a83353ddcee16d2e80dc56c968\" id=6ec0d6f8-ac91-4af4-b556-1c65a88cdcfe name=/runtime.v1.RuntimeService/StopPodSandbox","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-infrastructure"},"sequence":1746546273924327080},"systemd":{"t":{"BOOT_ID":"274588dcf3fa4b889eb852e04ce2da6b","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/bin/crio","COMM":"crio","EXE":"/usr/bin/crio","GID":"0","MACHINE_ID":"42ce264bc9a44b0e887485fc1297c3d3","PID":"2481","SELINUX_CONTEXT":"system_u:system_r:container_runtime_t:s0","STREAM_ID":"0fa7e533314547b8a24544289b1279e2","SYSTEMD_CGROUP":"/system.slice/crio.service","SYSTEMD_INVOCATION_ID":"11e1751345034868ab781f1b98f304ed","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"crio.service","TRANSPORT":"stdout","UID":"0"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"crio"}},"tag":".journal.system","time":"2025-05-05T06:18:09+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"CV9FppYBjyXErPejM0Ra","_score":0.012046979,"_source":{"@timestamp":"2025-05-05T06:18:09.011810Z","_RUNTIME_SCOPE":"system","_STREAM_ID":"0fa7e533314547b8a24544289b1279e2","_SYSTEMD_INVOCATION_ID":"11e1751345034868ab781f1b98f304ed","hostname":"master-1","level":"info","log_source":"node","log_type":"infrastructure","message":"time=\"2025-05-05 06:18:09.011783488Z\" level=info msg=\"Got pod network &{Name:collector-dczxq Namespace:openshift-logging ID:6eaaa1a0122eeed9f8ed23d8d75208891f9349a83353ddcee16d2e80dc56c968 UID:bfa1f07e-af16-489a-86aa-aa0c678ddef1 NetNS:/var/run/netns/36e56e9e-e100-453d-86c1-d6863ecc816f Networks:[{Name:multus-cni-network Ifname:eth0}] RuntimeConfig:map[multus-cni-network:{IP: MAC: PortMappings:[] Bandwidth: IpRanges:[] CgroupPath:kubepods-burstable-podbfa1f07e_af16_489a_86aa_aa0c678ddef1.slice}] Aliases:map[]}\"","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-infrastructure"},"sequence":1746546273924453613},"systemd":{"t":{"BOOT_ID":"274588dcf3fa4b889eb852e04ce2da6b","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/bin/crio","COMM":"crio","EXE":"/usr/bin/crio","GID":"0","MACHINE_ID":"42ce264bc9a44b0e887485fc1297c3d3","PID":"2481","SELINUX_CONTEXT":"system_u:system_r:container_runtime_t:s0","STREAM_ID":"0fa7e533314547b8a24544289b1279e2","SYSTEMD_CGROUP":"/system.slice/crio.service","SYSTEMD_INVOCATION_ID":"11e1751345034868ab781f1b98f304ed","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"crio.service","TRANSPORT":"stdout","UID":"0"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"crio"}},"tag":".journal.system","time":"2025-05-05T06:18:09+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"Cl9FppYBjyXErPejM0Ra","_score":0.012046979,"_source":{"@timestamp":"2025-05-05T06:18:09.011937Z","_RUNTIME_SCOPE":"system","_STREAM_ID":"0fa7e533314547b8a24544289b1279e2","_SYSTEMD_INVOCATION_ID":"11e1751345034868ab781f1b98f304ed","hostname":"master-1","level":"info","log_source":"node","log_type":"infrastructure","message":"time=\"2025-05-05 06:18:09.011917780Z\" level=info msg=\"Deleting pod openshift-logging_collector-dczxq from CNI network \\\"multus-cni-network\\\" (type=multus-shim)\"","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0_415-infrastructure"},"sequence":1746546273924654500},"systemd":{"t":{"BOOT_ID":"274588dcf3fa4b889eb852e04ce2da6b","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/bin/crio","COMM":"crio","EXE":"/usr/bin/crio","GID":"0","MACHINE_ID":"42ce264bc9a44b0e887485fc1297c3d3","PID":"2481","SELINUX_CONTEXT":"system_u:system_r:container_runtime_t:s0","STREAM_ID":"0fa7e533314547b8a24544289b1279e2","SYSTEMD_CGROUP":"/system.slice/crio.service","SYSTEMD_INVOCATION_ID":"11e1751345034868ab781f1b98f304ed","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"crio.service","TRANSPORT":"stdout","UID":"0"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"crio"}},"tag":".journal.system","time":"2025-05-05T06:18:09+00:00"}}]}} ``` Loki: Application logs: ``` {"ts":"2025-05-06T15:48:48.237420949Z","line":"{\"@timestamp\":\"2025-05-06T11:22:18.301165436Z\",\"hostname\":\"worker-0\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.76/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:4c\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.169.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.76/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.76\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:4c\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://5ebe6cfeb054b528e3c048d15d7a12e40e474e1e3fb95b417a4478270fddbfb6\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_image_id\":\"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912\",\"container_name\":\"centos-logtest\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"namespace_id\":\"e2166d8f-cd66-4fec-b7fe-df5234e1507a\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"test-logging\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"namespace_name\":\"test-logging\",\"pod_id\":\"c603b0ef-f979-4a5f-9e69-9ba0d44d3808\",\"pod_ip\":\"10.131.0.76\",\"pod_name\":\"centos-logtest-kfr6v\",\"pod_owner\":\"ReplicationController/centos-logtest\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"{\\\"message\\\": \\\"MERGE_JSON_LOG=true\\\", \\\"level\\\": \\\"debug\\\",\\\"Layer1\\\": \\\"layer1 0\\\", \\\"layer2\\\": {\\\"name\\\":\\\"Layer2 1\\\", \\\"tips\\\":\\\"Decide by PRESERVE_JSON_LOG\\\"}, \\\"StringNumber\\\":\\\"10\\\", \\\"Number\\\": 10,\\\"foo.bar\\\":\\\"Dot Item\\\",\\\"{foobar}\\\":\\\"Brace Item\\\",\\\"[foobar]\\\":\\\"Bracket Item\\\", \\\"foo:bar\\\":\\\"Colon Item\\\",\\\"foo bar\\\":\\\"Space Item\\\" }\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-application\"},\"sequence\":1746546487650948275}}"},{"ts":"2025-05-06T15:48:48.237392822Z","line":"{\"@timestamp\":\"2025-05-06T11:22:17.299526498Z\",\"hostname\":\"worker-0\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.76/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:4c\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.169.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.76/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.76\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:4c\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://5ebe6cfeb054b528e3c048d15d7a12e40e474e1e3fb95b417a4478270fddbfb6\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_image_id\":\"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912\",\"container_name\":\"centos-logtest\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"namespace_id\":\"e2166d8f-cd66-4fec-b7fe-df5234e1507a\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"test-logging\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"namespace_name\":\"test-logging\",\"pod_id\":\"c603b0ef-f979-4a5f-9e69-9ba0d44d3808\",\"pod_ip\":\"10.131.0.76\",\"pod_name\":\"centos-logtest-kfr6v\",\"pod_owner\":\"ReplicationController/centos-logtest\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"{\\\"message\\\": \\\"MERGE_JSON_LOG=true\\\", \\\"level\\\": \\\"debug\\\",\\\"Layer1\\\": \\\"layer1 0\\\", \\\"layer2\\\": {\\\"name\\\":\\\"Layer2 1\\\", \\\"tips\\\":\\\"Decide by PRESERVE_JSON_LOG\\\"}, \\\"StringNumber\\\":\\\"10\\\", \\\"Number\\\": 10,\\\"foo.bar\\\":\\\"Dot Item\\\",\\\"{foobar}\\\":\\\"Brace Item\\\",\\\"[foobar]\\\":\\\"Bracket Item\\\", \\\"foo:bar\\\":\\\"Colon Item\\\",\\\"foo bar\\\":\\\"Space Item\\\" }\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-application\"},\"sequence\":1746546487650895323}}"},{"ts":"2025-05-06T15:48:48.237361785Z","line":"{\"@timestamp\":\"2025-05-06T11:22:16.297948698Z\",\"hostname\":\"worker-0\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.76/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:4c\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.169.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.76/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.76\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:4c\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://5ebe6cfeb054b528e3c048d15d7a12e40e474e1e3fb95b417a4478270fddbfb6\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_image_id\":\"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912\",\"container_name\":\"centos-logtest\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"namespace_id\":\"e2166d8f-cd66-4fec-b7fe-df5234e1507a\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"test-logging\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"namespace_name\":\"test-logging\",\"pod_id\":\"c603b0ef-f979-4a5f-9e69-9ba0d44d3808\",\"pod_ip\":\"10.131.0.76\",\"pod_name\":\"centos-logtest-kfr6v\",\"pod_owner\":\"ReplicationController/centos-logtest\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"{\\\"message\\\": \\\"MERGE_JSON_LOG=true\\\", \\\"level\\\": \\\"debug\\\",\\\"Layer1\\\": \\\"layer1 0\\\", \\\"layer2\\\": {\\\"name\\\":\\\"Layer2 1\\\", \\\"tips\\\":\\\"Decide by PRESERVE_JSON_LOG\\\"}, \\\"StringNumber\\\":\\\"10\\\", \\\"Number\\\": 10,\\\"foo.bar\\\":\\\"Dot Item\\\",\\\"{foobar}\\\":\\\"Brace Item\\\",\\\"[foobar]\\\":\\\"Bracket Item\\\", \\\"foo:bar\\\":\\\"Colon Item\\\",\\\"foo bar\\\":\\\"Space Item\\\" }\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-application\"},\"sequence\":1746546487650842149}}"},{"ts":"2025-05-06T15:48:48.237326939Z","line":"{\"@timestamp\":\"2025-05-06T11:22:15.296409292Z\",\"hostname\":\"worker-0\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.76/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:4c\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.169.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.76/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.76\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:4c\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://5ebe6cfeb054b528e3c048d15d7a12e40e474e1e3fb95b417a4478270fddbfb6\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_image_id\":\"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912\",\"container_name\":\"centos-logtest\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"namespace_id\":\"e2166d8f-cd66-4fec-b7fe-df5234e1507a\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"test-logging\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"namespace_name\":\"test-logging\",\"pod_id\":\"c603b0ef-f979-4a5f-9e69-9ba0d44d3808\",\"pod_ip\":\"10.131.0.76\",\"pod_name\":\"centos-logtest-kfr6v\",\"pod_owner\":\"ReplicationController/centos-logtest\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"{\\\"message\\\": \\\"MERGE_JSON_LOG=true\\\", \\\"level\\\": \\\"debug\\\",\\\"Layer1\\\": \\\"layer1 0\\\", \\\"layer2\\\": {\\\"name\\\":\\\"Layer2 1\\\", \\\"tips\\\":\\\"Decide by PRESERVE_JSON_LOG\\\"}, \\\"StringNumber\\\":\\\"10\\\", \\\"Number\\\": 10,\\\"foo.bar\\\":\\\"Dot Item\\\",\\\"{foobar}\\\":\\\"Brace Item\\\",\\\"[foobar]\\\":\\\"Bracket Item\\\", \\\"foo:bar\\\":\\\"Colon Item\\\",\\\"foo bar\\\":\\\"Space Item\\\" }\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-application\"},\"sequence\":1746546487650787985}}"},{"ts":"2025-05-06T15:48:48.23729899Z","line":"{\"@timestamp\":\"2025-05-06T11:22:14.294916440Z\",\"hostname\":\"worker-0\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.76/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:4c\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.169.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.76/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.76\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:4c\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://5ebe6cfeb054b528e3c048d15d7a12e40e474e1e3fb95b417a4478270fddbfb6\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_image_id\":\"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912\",\"container_name\":\"centos-logtest\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"namespace_id\":\"e2166d8f-cd66-4fec-b7fe-df5234e1507a\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"test-logging\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"namespace_name\":\"test-logging\",\"pod_id\":\"c603b0ef-f979-4a5f-9e69-9ba0d44d3808\",\"pod_ip\":\"10.131.0.76\",\"pod_name\":\"centos-logtest-kfr6v\",\"pod_owner\":\"ReplicationController/centos-logtest\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"{\\\"message\\\": \\\"MERGE_JSON_LOG=true\\\", \\\"level\\\": \\\"debug\\\",\\\"Layer1\\\": \\\"layer1 0\\\", \\\"layer2\\\": {\\\"name\\\":\\\"Layer2 1\\\", \\\"tips\\\":\\\"Decide by PRESERVE_JSON_LOG\\\"}, \\\"StringNumber\\\":\\\"10\\\", \\\"Number\\\": 10,\\\"foo.bar\\\":\\\"Dot Item\\\",\\\"{foobar}\\\":\\\"Brace Item\\\",\\\"[foobar]\\\":\\\"Bracket Item\\\", \\\"foo:bar\\\":\\\"Colon Item\\\",\\\"foo bar\\\":\\\"Space Item\\\" }\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-application\"},\"sequence\":1746546487650727905}}"}]}],"stats":{"summary":{"bytesProcessedPerSecond":827073876,"linesProcessedPerSecond":353342,"totalBytesProcessed":285719695,"totalLinesProcessed":122065,"execTime":0.345458493,"queueTime":0.00005,"subqueries":0,"totalEntriesReturned":100,"splits":1,"shards":0},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0}}},"ingester":{"totalReached":1,"totalChunksMatched":10,"totalBatches":1,"totalLinesSent":100,"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":553508,"headChunkLines":251,"decompressedBytes":285166187,"decompressedLines":121814,"compressedBytes":5909255,"totalDuplicates":0}}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0}}}} ``` Audit logs: ``` {"ts":"2025-05-06T15:52:03.686601817Z","line":"{\"@timestamp\":\"2025-05-06T15:52:03.652244565Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"custom-account-openshift-machine-config-operator\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"machine-config-operator/openshift-machine-config-operator\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"e2deed88-602f-484e-b743-7980142ae667\",\"hostname\":\"master-2\",\"k8s_audit_level\":\"Metadata\",\"kind\":\"Event\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"Metadata\",\"log_source\":\"kubeAPI\",\"log_type\":\"audit\",\"objectRef\":{\"apiGroup\":\"config.openshift.io\",\"apiVersion\":\"v1\",\"resource\":\"clusterversions\"},\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-audit\"},\"sequence\":1746546723652689528},\"requestReceivedTimestamp\":\"2025-05-06T15:43:37.620782Z\",\"requestURI\":\"/apis/config.openshift.io/v1/clusterversions?allowWatchBookmarks=true\u0026resourceVersion=1846308\u0026timeout=8m26s\u0026timeoutSeconds=506\u0026watch=true\",\"responseStatus\":{\"code\":200,\"metadata\":{}},\"sourceIPs\":[\"10.130.0.247\"],\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-06T15:52:03.625111Z\",\"user\":{\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"machine-config-operator-59cd87895f-mk4tt\"],\"authentication.kubernetes.io/pod-uid\":[\"f251a638-e5a7-4645-9906-a36920d8a378\"]},\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-machine-config-operator\",\"system:authenticated\"],\"uid\":\"7a8bac61-5ed4-4f13-868d-f8258ea7799b\",\"username\":\"system:serviceaccount:openshift-machine-config-operator:machine-config-operator\"},\"userAgent\":\"machine-config-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format/config-shared-informer\",\"verb\":\"watch\"}"},{"ts":"2025-05-06T15:52:03.68654587Z","line":"{\"@timestamp\":\"2025-05-06T15:52:03.638396688Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:discovery\\\" of ClusterRole \\\"system:discovery\\\" to Group \\\"system:authenticated\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"7bfd0138-2c39-4f28-ab29-da87d3ded960\",\"hostname\":\"master-2\",\"kind\":\"Event\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"Metadata\",\"log_source\":\"openshiftAPI\",\"log_type\":\"audit\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-audit\"},\"sequence\":1746546723639182967},\"openshift_audit_level\":\"Metadata\",\"requestReceivedTimestamp\":\"2025-05-06T15:52:03.467736Z\",\"requestURI\":\"/openapi/v3\",\"responseStatus\":{\"code\":200,\"metadata\":{}},\"sourceIPs\":[\"10.128.0.2\"],\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-06T15:52:03.468617Z\",\"user\":{\"groups\":[\"system:authenticated\"],\"username\":\"system:aggregator\"},\"verb\":\"get\"}"},{"ts":"2025-05-06T15:52:03.686490527Z","line":"{\"@timestamp\":\"2025-05-06T15:52:03.638390850Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"cluster-status-binding\\\" of ClusterRole \\\"cluster-status\\\" to Group \\\"system:authenticated\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"4762ccf5-4145-4eba-9706-e2bca9288e25\",\"hostname\":\"master-2\",\"kind\":\"Event\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"Metadata\",\"log_source\":\"openshiftAPI\",\"log_type\":\"audit\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-audit\"},\"sequence\":1746546723639139916},\"openshift_audit_level\":\"Metadata\",\"requestReceivedTimestamp\":\"2025-05-06T15:52:03.383307Z\",\"requestURI\":\"/openapi/v2\",\"responseStatus\":{\"code\":304,\"metadata\":{}},\"sourceIPs\":[\"10.128.0.2\"],\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-06T15:52:03.384194Z\",\"user\":{\"groups\":[\"system:authenticated\"],\"username\":\"system:aggregator\"},\"verb\":\"get\"}"},{"ts":"2025-05-06T15:52:03.686435464Z","line":"{\"@timestamp\":\"2025-05-06T15:52:03.638384457Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"cluster-status-binding\\\" of ClusterRole \\\"cluster-status\\\" to Group \\\"system:authenticated\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"9e59d68f-d765-4f95-a044-1df051663cfa\",\"hostname\":\"master-2\",\"kind\":\"Event\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"Metadata\",\"log_source\":\"openshiftAPI\",\"log_type\":\"audit\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-audit\"},\"sequence\":1746546723639093713},\"openshift_audit_level\":\"Metadata\",\"requestReceivedTimestamp\":\"2025-05-06T15:52:03.379527Z\",\"requestURI\":\"/openapi/v2\",\"responseStatus\":{\"code\":304,\"metadata\":{}},\"sourceIPs\":[\"10.128.0.2\"],\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-06T15:52:03.380249Z\",\"user\":{\"groups\":[\"system:authenticated\"],\"username\":\"system:aggregator\"},\"verb\":\"get\"}"},{"ts":"2025-05-06T15:52:03.686376684Z","line":"{\"@timestamp\":\"2025-05-06T15:52:03.638378414Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"cluster-status-binding\\\" of ClusterRole \\\"cluster-status\\\" to Group \\\"system:authenticated\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"ebfcacb0-f711-4bb1-8f5f-bd444ab22260\",\"hostname\":\"master-2\",\"kind\":\"Event\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"Metadata\",\"log_source\":\"openshiftAPI\",\"log_type\":\"audit\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-audit\"},\"sequence\":1746546723639047645},\"openshift_audit_level\":\"Metadata\",\"requestReceivedTimestamp\":\"2025-05-06T15:52:03.369453Z\",\"requestURI\":\"/openapi/v2\",\"responseStatus\":{\"code\":304,\"metadata\":{}},\"sourceIPs\":[\"10.128.0.2\"],\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-06T15:52:03.370258Z\",\"user\":{\"groups\":[\"system:authenticated\"],\"username\":\"system:aggregator\"},\"verb\":\"get\"}"},{"ts":"2025-05-06T15:52:03.686324121Z","line":"{\"@timestamp\":\"2025-05-06T15:52:03.638372465Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:discovery\\\" of ClusterRole \\\"system:discovery\\\" to Group \\\"system:authenticated\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"005901b4-ff11-47f8-85da-24e8abcf4bda\",\"hostname\":\"master-2\",\"kind\":\"Event\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"Metadata\",\"log_source\":\"openshiftAPI\",\"log_type\":\"audit\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-audit\"},\"sequence\":1746546723639000655},\"openshift_audit_level\":\"Metadata\",\"requestReceivedTimestamp\":\"2025-05-06T15:52:03.291543Z\",\"requestURI\":\"/openapi/v3\",\"responseStatus\":{\"code\":200,\"metadata\":{}},\"sourceIPs\":[\"10.128.0.2\"],\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-06T15:52:03.292338Z\",\"user\":{\"groups\":[\"system:authenticated\"],\"username\":\"system:aggregator\"},\"verb\":\"get\"}"},{"ts":"2025-05-06T15:52:03.68626784Z","line":"{\"@timestamp\":\"2025-05-06T15:52:03.638366215Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:discovery\\\" of ClusterRole \\\"system:discovery\\\" to Group \\\"system:authenticated\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"9adf781b-15f0-476c-8665-bc59734e6fa0\",\"hostname\":\"master-2\",\"kind\":\"Event\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"Metadata\",\"log_source\":\"openshiftAPI\",\"log_type\":\"audit\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-audit\"},\"sequence\":1746546723638953124},\"openshift_audit_level\":\"Metadata\",\"requestReceivedTimestamp\":\"2025-05-06T15:52:03.261758Z\",\"requestURI\":\"/openapi/v3\",\"responseStatus\":{\"code\":200,\"metadata\":{}},\"sourceIPs\":[\"10.128.0.2\"],\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-06T15:52:03.266103Z\",\"user\":{\"groups\":[\"system:authenticated\"],\"username\":\"system:aggregator\"},\"verb\":\"get\"}"},{"ts":"2025-05-06T15:52:03.686203761Z","line":"{\"@timestamp\":\"2025-05-06T15:52:03.638359221Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"cluster-status-binding\\\" of ClusterRole \\\"cluster-status\\\" to Group \\\"system:authenticated\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"92a2c5da-f6df-4907-9707-487a7509ded8\",\"hostname\":\"master-2\",\"kind\":\"Event\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"Metadata\",\"log_source\":\"openshiftAPI\",\"log_type\":\"audit\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-audit\"},\"sequence\":1746546723638905627},\"openshift_audit_level\":\"Metadata\",\"requestReceivedTimestamp\":\"2025-05-06T15:52:03.233519Z\",\"requestURI\":\"/openapi/v2\",\"responseStatus\":{\"code\":304,\"metadata\":{}},\"sourceIPs\":[\"10.128.0.2\"],\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-06T15:52:03.236788Z\",\"user\":{\"groups\":[\"system:authenticated\"],\"username\":\"system:aggregator\"},\"verb\":\"get\"}"},{"ts":"2025-05-06T15:52:03.686148228Z","line":"{\"@timestamp\":\"2025-05-06T15:52:03.638352231Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:discovery\\\" of ClusterRole \\\"system:discovery\\\" to Group \\\"system:authenticated\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"de849c07-d0a9-4f17-b180-f31819647d4a\",\"hostname\":\"master-2\",\"kind\":\"Event\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"Metadata\",\"log_source\":\"openshiftAPI\",\"log_type\":\"audit\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-audit\"},\"sequence\":1746546723638836893},\"openshift_audit_level\":\"Metadata\",\"requestReceivedTimestamp\":\"2025-05-06T15:52:03.439207Z\",\"requestURI\":\"/openapi/v3\",\"responseStatus\":{\"code\":200,\"metadata\":{}},\"sourceIPs\":[\"10.128.0.2\"],\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-06T15:52:03.443593Z\",\"user\":{\"groups\":[\"system:authenticated\"],\"username\":\"system:aggregator\"},\"verb\":\"get\"}"},{"ts":"2025-05-06T15:52:03.686096945Z","line":"{\"@timestamp\":\"2025-05-06T15:52:03.638331690Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:openshift:discovery\\\" of ClusterRole \\\"system:openshift:discovery\\\" to Group \\\"system:authenticated\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"0e1ca31f-bc90-47ff-8b60-6e175c70ce81\",\"hostname\":\"master-2\",\"kind\":\"Event\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"Metadata\",\"log_source\":\"openshiftAPI\",\"log_type\":\"audit\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-audit\"},\"sequence\":1746546723638762625},\"openshift_audit_level\":\"Metadata\",\"requestReceivedTimestamp\":\"2025-05-06T15:52:03.421913Z\",\"requestURI\":\"/openapi/v2\",\"responseStatus\":{\"code\":304,\"metadata\":{}},\"sourceIPs\":[\"10.128.0.2\"],\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-06T15:52:03.432400Z\",\"user\":{\"groups\":[\"system:authenticated\"],\"username\":\"system:aggregator\"},\"verb\":\"get\"}"},{"ts":"2025-05-06T15:52:03.686027642Z","line":"{\"@timestamp\":\"2025-05-06T15:52:03.519938613Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by RoleBinding \\\"control-plane-machine-set-operator/openshift-machine-api\\\" of Role \\\"control-plane-machine-set-operator\\\" to ServiceAccount \\\"control-plane-machine-set-operator/openshift-machine-api\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"d6ef542a-de45-4c3c-8aba-4efc23be1cf6\",\"hostname\":\"master-2\",\"k8s_audit_level\":\"Metadata\",\"kind\":\"Event\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"Metadata\",\"log_source\":\"kubeAPI\",\"log_type\":\"audit\",\"objectRef\":{\"apiGroup\":\"machine.openshift.io\",\"apiVersion\":\"v1\",\"namespace\":\"openshift-machine-api\",\"resource\":\"controlplanemachinesets\"},\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-audit\"},\"sequence\":1746546723520293289},\"requestReceivedTimestamp\":\"2025-05-06T15:52:03.504672Z\",\"requestURI\":\"/apis/machine.openshift.io/v1/namespaces/openshift-machine-api/controlplanemachinesets?allowWatchBookmarks=true\u0026resourceVersion=1849492\u0026timeoutSeconds=509\u0026watch=true\",\"responseStatus\":{\"code\":200,\"metadata\":{}},\"sourceIPs\":[\"10.130.1.10\"],\"stage\":\"ResponseStarted\",\"stageTimestamp\":\"2025-05-06T15:52:03.505570Z\",\"user\":{\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"control-plane-machine-set-operator-5f6699b5d-pgkl2\"],\"authentication.kubernetes.io/pod-uid\":[\"e7d42522-c3c8-4f01-8908-c007d32aac4c\"]},\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-machine-api\",\"system:authenticated\"],\"uid\":\"3a9de2ae-f2b3-4e53-a89d-f7795dadb9e7\",\"username\":\"system:serviceaccount:openshift-machine-api:control-plane-machine-set-operator\"},\"userAgent\":\"manager/v0.0.0 (linux/ppc64le) kubernetes/$Format\",\"verb\":\"watch\"}"}]}],"stats":{"summary":{"bytesProcessedPerSecond":411709507,"linesProcessedPerSecond":324231,"totalBytesProcessed":73592606,"totalLinesProcessed":57956,"execTime":0.178748862,"queueTime":0.000043,"subqueries":0,"totalEntriesReturned":100,"splits":1,"shards":0},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0}}},"ingester":{"totalReached":1,"totalChunksMatched":4,"totalBatches":1,"totalLinesSent":100,"store":{"totalChunksRef":8,"totalChunksDownloaded":8,"chunksDownloadTime":4194202,"chunk":{"headChunkBytes":728870,"headChunkLines":841,"decompressedBytes":72863736,"decompressedLines":57115,"compressedBytes":62159888,"totalDuplicates":0}}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0}}}} ``` Infrastructure logs: ``` {"ts":"2025-05-06T15:52:16.50165201Z","line":"{\"@timestamp\":\"2025-05-05T23:47:25.556594Z\",\"_RUNTIME_SCOPE\":\"system\",\"_STREAM_ID\":\"3942dd8c215a4489bb428b9e109bc4b1\",\"_SYSTEMD_INVOCATION_ID\":\"fcd08ec90c23487fa33ea7ed5e9e6df4\",\"hostname\":\"master-2\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"info\",\"log_source\":\"node\",\"log_type\":\"infrastructure\",\"message\":\"time=\\\"2025-05-05 23:47:25.556414820Z\\\" level=info msg=\\\"Got pod network \u0026{Name:redhat-operators-pn9t8 Namespace:openshift-marketplace ID:7acff7fbf3068e7e1340a36bcc199df3e69eed39ff7887d9b5b604a64fdc6e71 UID:b8865167-68b4-448e-8a62-b1332fd8aded NetNS:/var/run/netns/1e48e1ff-b786-4bab-b41a-149d055591d4 Networks:[{Name:multus-cni-network Ifname:eth0}] RuntimeConfig:map[multus-cni-network:{IP: MAC: PortMappings:[] Bandwidth:\u003cnil\u003e IpRanges:[] CgroupPath:kubepods-burstable-podb8865167_68b4_448e_8a62_b1332fd8aded.slice}] Aliases:map[]}\\\"\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-infrastructure\"},\"sequence\":1746546728215510656},\"systemd\":{\"t\":{\"BOOT_ID\":\"1d807b865e62428b903a6c2ea9d70173\",\"CAP_EFFECTIVE\":\"1ffffffffff\",\"CMDLINE\":\"/usr/bin/crio\",\"COMM\":\"crio\",\"EXE\":\"/usr/bin/crio\",\"GID\":\"0\",\"MACHINE_ID\":\"689138b405714a06a820f2760e64ac76\",\"PID\":\"2480\",\"SELINUX_CONTEXT\":\"system_u:system_r:container_runtime_t:s0\",\"STREAM_ID\":\"3942dd8c215a4489bb428b9e109bc4b1\",\"SYSTEMD_CGROUP\":\"/system.slice/crio.service\",\"SYSTEMD_INVOCATION_ID\":\"fcd08ec90c23487fa33ea7ed5e9e6df4\",\"SYSTEMD_SLICE\":\"system.slice\",\"SYSTEMD_UNIT\":\"crio.service\",\"TRANSPORT\":\"stdout\",\"UID\":\"0\"},\"u\":{\"SYSLOG_FACILITY\":\"3\",\"SYSLOG_IDENTIFIER\":\"crio\"}},\"time\":\"2025-05-05T23:47:25+00:00\"}"},{"ts":"2025-05-06T15:52:16.501622229Z","line":"{\"@timestamp\":\"2025-05-05T23:47:25.555658Z\",\"_RUNTIME_SCOPE\":\"system\",\"_STREAM_ID\":\"3942dd8c215a4489bb428b9e109bc4b1\",\"_SYSTEMD_INVOCATION_ID\":\"fcd08ec90c23487fa33ea7ed5e9e6df4\",\"hostname\":\"master-2\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"info\",\"log_source\":\"node\",\"log_type\":\"infrastructure\",\"message\":\"time=\\\"2025-05-05 23:47:25.555624699Z\\\" level=info msg=\\\"Stopping pod sandbox: 7acff7fbf3068e7e1340a36bcc199df3e69eed39ff7887d9b5b604a64fdc6e71\\\" id=1096a007-c8f1-4a0e-8796-632e1a9e8ee1 name=/runtime.v1.RuntimeService/StopPodSandbox\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-infrastructure\"},\"sequence\":1746546728215402623},\"systemd\":{\"t\":{\"BOOT_ID\":\"1d807b865e62428b903a6c2ea9d70173\",\"CAP_EFFECTIVE\":\"1ffffffffff\",\"CMDLINE\":\"/usr/bin/crio\",\"COMM\":\"crio\",\"EXE\":\"/usr/bin/crio\",\"GID\":\"0\",\"MACHINE_ID\":\"689138b405714a06a820f2760e64ac76\",\"PID\":\"2480\",\"SELINUX_CONTEXT\":\"system_u:system_r:container_runtime_t:s0\",\"STREAM_ID\":\"3942dd8c215a4489bb428b9e109bc4b1\",\"SYSTEMD_CGROUP\":\"/system.slice/crio.service\",\"SYSTEMD_INVOCATION_ID\":\"fcd08ec90c23487fa33ea7ed5e9e6df4\",\"SYSTEMD_SLICE\":\"system.slice\",\"SYSTEMD_UNIT\":\"crio.service\",\"TRANSPORT\":\"stdout\",\"UID\":\"0\"},\"u\":{\"SYSLOG_FACILITY\":\"3\",\"SYSLOG_IDENTIFIER\":\"crio\"}},\"time\":\"2025-05-05T23:47:25+00:00\"}"},{"ts":"2025-05-06T15:52:16.501520104Z","line":"{\"@timestamp\":\"2025-05-05T23:47:25.554762Z\",\"_RUNTIME_SCOPE\":\"system\",\"_STREAM_ID\":\"3942dd8c215a4489bb428b9e109bc4b1\",\"_SYSTEMD_INVOCATION_ID\":\"fcd08ec90c23487fa33ea7ed5e9e6df4\",\"hostname\":\"master-2\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"info\",\"log_source\":\"node\",\"log_type\":\"infrastructure\",\"message\":\"time=\\\"2025-05-05 23:47:25.554464417Z\\\" level=info msg=\\\"Stopped container ef7d4dd4d3d35d9f34c4bc2c2fb61d3bb99111a91a6187cec2b24edfd99d3f75: openshift-marketplace/redhat-operators-pn9t8/registry-server\\\" id=d8ada0fe-39a2-4094-a030-9e6b1dfa5634 name=/runtime.v1.RuntimeService/StopContainer\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-infrastructure\"},\"sequence\":1746546728215295530},\"systemd\":{\"t\":{\"BOOT_ID\":\"1d807b865e62428b903a6c2ea9d70173\",\"CAP_EFFECTIVE\":\"1ffffffffff\",\"CMDLINE\":\"/usr/bin/crio\",\"COMM\":\"crio\",\"EXE\":\"/usr/bin/crio\",\"GID\":\"0\",\"MACHINE_ID\":\"689138b405714a06a820f2760e64ac76\",\"PID\":\"2480\",\"SELINUX_CONTEXT\":\"system_u:system_r:container_runtime_t:s0\",\"STREAM_ID\":\"3942dd8c215a4489bb428b9e109bc4b1\",\"SYSTEMD_CGROUP\":\"/system.slice/crio.service\",\"SYSTEMD_INVOCATION_ID\":\"fcd08ec90c23487fa33ea7ed5e9e6df4\",\"SYSTEMD_SLICE\":\"system.slice\",\"SYSTEMD_UNIT\":\"crio.service\",\"TRANSPORT\":\"stdout\",\"UID\":\"0\"},\"u\":{\"SYSLOG_FACILITY\":\"3\",\"SYSLOG_IDENTIFIER\":\"crio\"}},\"time\":\"2025-05-05T23:47:25+00:00\"}"},{"ts":"2025-05-06T15:52:16.501490366Z","line":"{\"@timestamp\":\"2025-05-05T23:47:25.394717Z\",\"INVOCATION_ID\":\"fe6fb1ff22e84da5acf35fdef89d7ac8\",\"TID\":\"1\",\"_RUNTIME_SCOPE\":\"system\",\"hostname\":\"master-2\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"info\",\"log_source\":\"node\",\"log_type\":\"infrastructure\",\"message\":\"crio-conmon-ef7d4dd4d3d35d9f34c4bc2c2fb61d3bb99111a91a6187cec2b24edfd99d3f75.scope: Deactivated successfully.\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-infrastructure\"},\"sequence\":1746546728215184851},\"systemd\":{\"t\":{\"BOOT_ID\":\"1d807b865e62428b903a6c2ea9d70173\",\"CAP_EFFECTIVE\":\"1ffffffffff\",\"CMDLINE\":\"/usr/lib/systemd/systemd --switched-root --system --deserialize 28\",\"COMM\":\"systemd\",\"EXE\":\"/usr/lib/systemd/systemd\",\"GID\":\"0\",\"MACHINE_ID\":\"689138b405714a06a820f2760e64ac76\",\"PID\":\"1\",\"SELINUX_CONTEXT\":\"system_u:system_r:init_t:s0\",\"SYSTEMD_CGROUP\":\"/init.scope\",\"SYSTEMD_SLICE\":\"-.slice\",\"SYSTEMD_UNIT\":\"init.scope\",\"TRANSPORT\":\"journal\",\"UID\":\"0\"},\"u\":{\"CODE_FILE\":\"src/core/unit.c\",\"CODE_FUNCTION\":\"unit_log_success\",\"CODE_LINE\":\"5593\",\"MESSAGE_ID\":\"7ad2d189f7e94e70a38c781354912448\",\"SYSLOG_FACILITY\":\"3\",\"SYSLOG_IDENTIFIER\":\"systemd\",\"UNIT\":\"crio-conmon-ef7d4dd4d3d35d9f34c4bc2c2fb61d3bb99111a91a6187cec2b24edfd99d3f75.scope\"}},\"time\":\"2025-05-05T23:47:25+00:00\"}"},{"ts":"2025-05-06T15:52:16.501462427Z","line":"{\"@timestamp\":\"2025-05-05T23:47:25.391166Z\",\"CPU_USAGE_NSEC\":\"6860245000\",\"INVOCATION_ID\":\"bdc3ab42092b4e00b409eaee93a9f9c1\",\"TID\":\"1\",\"_RUNTIME_SCOPE\":\"system\",\"hostname\":\"master-2\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"info\",\"log_source\":\"node\",\"log_type\":\"infrastructure\",\"message\":\"crio-ef7d4dd4d3d35d9f34c4bc2c2fb61d3bb99111a91a6187cec2b24edfd99d3f75.scope: Consumed 6.860s CPU time.\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-infrastructure\"},\"sequence\":1746546728215056742},\"systemd\":{\"t\":{\"BOOT_ID\":\"1d807b865e62428b903a6c2ea9d70173\",\"CAP_EFFECTIVE\":\"1ffffffffff\",\"CMDLINE\":\"/usr/lib/systemd/systemd --switched-root --system --deserialize 28\",\"COMM\":\"systemd\",\"EXE\":\"/usr/lib/systemd/systemd\",\"GID\":\"0\",\"MACHINE_ID\":\"689138b405714a06a820f2760e64ac76\",\"PID\":\"1\",\"SELINUX_CONTEXT\":\"system_u:system_r:init_t:s0\",\"SYSTEMD_CGROUP\":\"/init.scope\",\"SYSTEMD_SLICE\":\"-.slice\",\"SYSTEMD_UNIT\":\"init.scope\",\"TRANSPORT\":\"journal\",\"UID\":\"0\"},\"u\":{\"CODE_FILE\":\"src/core/unit.c\",\"CODE_FUNCTION\":\"unit_log_resources\",\"CODE_LINE\":\"2483\",\"MESSAGE_ID\":\"ae8f7b866b0347b9af31fe1c80b127c0\",\"SYSLOG_FACILITY\":\"3\",\"SYSLOG_IDENTIFIER\":\"systemd\",\"UNIT\":\"crio-ef7d4dd4d3d35d9f34c4bc2c2fb61d3bb99111a91a6187cec2b24edfd99d3f75.scope\"}},\"time\":\"2025-05-05T23:47:25+00:00\"}"},{"ts":"2025-05-06T15:52:16.501430692Z","line":"{\"@timestamp\":\"2025-05-05T23:47:25.390692Z\",\"INVOCATION_ID\":\"bdc3ab42092b4e00b409eaee93a9f9c1\",\"TID\":\"1\",\"_RUNTIME_SCOPE\":\"system\",\"hostname\":\"master-2\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"info\",\"log_source\":\"node\",\"log_type\":\"infrastructure\",\"message\":\"crio-ef7d4dd4d3d35d9f34c4bc2c2fb61d3bb99111a91a6187cec2b24edfd99d3f75.scope: Deactivated successfully.\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-infrastructure\"},\"sequence\":1746546728214937799},\"systemd\":{\"t\":{\"BOOT_ID\":\"1d807b865e62428b903a6c2ea9d70173\",\"CAP_EFFECTIVE\":\"1ffffffffff\",\"CMDLINE\":\"/usr/lib/systemd/systemd --switched-root --system --deserialize 28\",\"COMM\":\"systemd\",\"EXE\":\"/usr/lib/systemd/systemd\",\"GID\":\"0\",\"MACHINE_ID\":\"689138b405714a06a820f2760e64ac76\",\"PID\":\"1\",\"SELINUX_CONTEXT\":\"system_u:system_r:init_t:s0\",\"SYSTEMD_CGROUP\":\"/init.scope\",\"SYSTEMD_SLICE\":\"-.slice\",\"SYSTEMD_UNIT\":\"init.scope\",\"TRANSPORT\":\"journal\",\"UID\":\"0\"},\"u\":{\"CODE_FILE\":\"src/core/unit.c\",\"CODE_FUNCTION\":\"unit_log_success\",\"CODE_LINE\":\"5593\",\"MESSAGE_ID\":\"7ad2d189f7e94e70a38c781354912448\",\"SYSLOG_FACILITY\":\"3\",\"SYSLOG_IDENTIFIER\":\"systemd\",\"UNIT\":\"crio-ef7d4dd4d3d35d9f34c4bc2c2fb61d3bb99111a91a6187cec2b24edfd99d3f75.scope\"}},\"time\":\"2025-05-05T23:47:25+00:00\"}"},{"ts":"2025-05-06T15:52:16.501387954Z","line":"{\"@timestamp\":\"2025-05-05T23:47:25.372571Z\",\"_RUNTIME_SCOPE\":\"system\",\"_STREAM_ID\":\"3942dd8c215a4489bb428b9e109bc4b1\",\"_SYSTEMD_INVOCATION_ID\":\"fcd08ec90c23487fa33ea7ed5e9e6df4\",\"hostname\":\"master-2\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"info\",\"log_source\":\"node\",\"log_type\":\"infrastructure\",\"message\":\"time=\\\"2025-05-05 23:47:25.372105130Z\\\" level=info msg=\\\"Stopping container: ef7d4dd4d3d35d9f34c4bc2c2fb61d3bb99111a91a6187cec2b24edfd99d3f75 (timeout: 2s)\\\" id=d8ada0fe-39a2-4094-a030-9e6b1dfa5634 name=/runtime.v1.RuntimeService/StopContainer\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-infrastructure\"},\"sequence\":1746546728214818960},\"systemd\":{\"t\":{\"BOOT_ID\":\"1d807b865e62428b903a6c2ea9d70173\",\"CAP_EFFECTIVE\":\"1ffffffffff\",\"CMDLINE\":\"/usr/bin/crio\",\"COMM\":\"crio\",\"EXE\":\"/usr/bin/crio\",\"GID\":\"0\",\"MACHINE_ID\":\"689138b405714a06a820f2760e64ac76\",\"PID\":\"2480\",\"SELINUX_CONTEXT\":\"system_u:system_r:container_runtime_t:s0\",\"STREAM_ID\":\"3942dd8c215a4489bb428b9e109bc4b1\",\"SYSTEMD_CGROUP\":\"/system.slice/crio.service\",\"SYSTEMD_INVOCATION_ID\":\"fcd08ec90c23487fa33ea7ed5e9e6df4\",\"SYSTEMD_SLICE\":\"system.slice\",\"SYSTEMD_UNIT\":\"crio.service\",\"TRANSPORT\":\"stdout\",\"UID\":\"0\"},\"u\":{\"SYSLOG_FACILITY\":\"3\",\"SYSLOG_IDENTIFIER\":\"crio\"}},\"time\":\"2025-05-05T23:47:25+00:00\"}"},{"ts":"2025-05-06T15:52:16.50133224Z","line":"{\"@timestamp\":\"2025-05-05T23:47:25.371808Z\",\"_RUNTIME_SCOPE\":\"system\",\"_STREAM_ID\":\"1ffb167e05bc4564a14ef8fed70cf057\",\"_SYSTEMD_INVOCATION_ID\":\"3d98d53d32c94d2a89532e1d10825bf5\",\"hostname\":\"master-2\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"info\",\"log_source\":\"node\",\"log_type\":\"infrastructure\",\"message\":\"I0505 23:47:25.371770 2543 kuberuntime_container.go:745] \\\"Killing container with a grace period\\\" pod=\\\"openshift-marketplace/redhat-operators-pn9t8\\\" podUID=\\\"b8865167-68b4-448e-8a62-b1332fd8aded\\\" containerName=\\\"registry-server\\\" containerID=\\\"cri-o://ef7d4dd4d3d35d9f34c4bc2c2fb61d3bb99111a91a6187cec2b24edfd99d3f75\\\" gracePeriod=2\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-infrastructure\"},\"sequence\":1746546728214710162},\"systemd\":{\"t\":{\"BOOT_ID\":\"1d807b865e62428b903a6c2ea9d70173\",\"CAP_EFFECTIVE\":\"1ffffffffff\",\"CMDLINE\":\"/usr/bin/kubelet --config=/etc/kubernetes/kubelet.conf --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig --container-runtime-endpoint=/var/run/crio/crio.sock --runtime-cgroups=/system.slice/crio.service --node-labels=node-role.kubernetes.io/control-plane,node-role.kubernetes.io/master,node.openshift.io/os_id=rhcos --node-ip=10.20.177.77 --minimum-container-ttl-duration=6m0s --cloud-provider= --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec --hostname-override= --provider-id= --register-with-taints=node-role.kubernetes.io/master=:NoSchedule --pod-infra-container-image=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:546b3635ab9f651032bf142e243809d64391ee7fcdec9c3d0636218c3dd7aa08 --system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=1Gi --v=2\",\"COMM\":\"kubelet\",\"EXE\":\"/usr/bin/kubelet\",\"GID\":\"0\",\"MACHINE_ID\":\"689138b405714a06a820f2760e64ac76\",\"PID\":\"2543\",\"SELINUX_CONTEXT\":\"system_u:system_r:kubelet_t:s0-s0:c0.c1023\",\"STREAM_ID\":\"1ffb167e05bc4564a14ef8fed70cf057\",\"SYSTEMD_CGROUP\":\"/system.slice/kubelet.service\",\"SYSTEMD_INVOCATION_ID\":\"3d98d53d32c94d2a89532e1d10825bf5\",\"SYSTEMD_SLICE\":\"system.slice\",\"SYSTEMD_UNIT\":\"kubelet.service\",\"TRANSPORT\":\"stdout\",\"UID\":\"0\"},\"u\":{\"SYSLOG_FACILITY\":\"3\",\"SYSLOG_IDENTIFIER\":\"kubenswrapper\"}},\"time\":\"2025-05-05T23:47:25+00:00\"}"},{"ts":"2025-05-06T15:52:16.498825097Z","line":"{\"@timestamp\":\"2025-05-05T23:47:24.063081Z\",\"_RUNTIME_SCOPE\":\"system\",\"_STREAM_ID\":\"1ffb167e05bc4564a14ef8fed70cf057\",\"_SYSTEMD_INVOCATION_ID\":\"3d98d53d32c94d2a89532e1d10825bf5\",\"hostname\":\"master-2\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"info\",\"log_source\":\"node\",\"log_type\":\"infrastructure\",\"message\":\"I0505 23:47:24.063013 2543 kubelet.go:2437] \\\"SyncLoop DELETE\\\" source=\\\"api\\\" pods=[\\\"openshift-marketplace/redhat-operators-pn9t8\\\"]\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-infrastructure\"},\"sequence\":1746546728210118768},\"systemd\":{\"t\":{\"BOOT_ID\":\"1d807b865e62428b903a6c2ea9d70173\",\"CAP_EFFECTIVE\":\"1ffffffffff\",\"CMDLINE\":\"/usr/bin/kubelet --config=/etc/kubernetes/kubelet.conf --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig --container-runtime-endpoint=/var/run/crio/crio.sock --runtime-cgroups=/system.slice/crio.service --node-labels=node-role.kubernetes.io/control-plane,node-role.kubernetes.io/master,node.openshift.io/os_id=rhcos --node-ip=10.20.177.77 --minimum-container-ttl-duration=6m0s --cloud-provider= --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec --hostname-override= --provider-id= --register-with-taints=node-role.kubernetes.io/master=:NoSchedule --pod-infra-container-image=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:546b3635ab9f651032bf142e243809d64391ee7fcdec9c3d0636218c3dd7aa08 --system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=1Gi --v=2\",\"COMM\":\"kubelet\",\"EXE\":\"/usr/bin/kubelet\",\"GID\":\"0\",\"MACHINE_ID\":\"689138b405714a06a820f2760e64ac76\",\"PID\":\"2543\",\"SELINUX_CONTEXT\":\"system_u:system_r:kubelet_t:s0-s0:c0.c1023\",\"STREAM_ID\":\"1ffb167e05bc4564a14ef8fed70cf057\",\"SYSTEMD_CGROUP\":\"/system.slice/kubelet.service\",\"SYSTEMD_INVOCATION_ID\":\"3d98d53d32c94d2a89532e1d10825bf5\",\"SYSTEMD_SLICE\":\"system.slice\",\"SYSTEMD_UNIT\":\"kubelet.service\",\"TRANSPORT\":\"stdout\",\"UID\":\"0\"},\"u\":{\"SYSLOG_FACILITY\":\"3\",\"SYSLOG_IDENTIFIER\":\"kubenswrapper\"}},\"time\":\"2025-05-05T23:47:24+00:00\"}"},{"ts":"2025-05-06T15:52:16.498801513Z","line":"{\"@timestamp\":\"2025-05-05T23:47:24.008843Z\",\"_RUNTIME_SCOPE\":\"system\",\"_STREAM_ID\":\"1ffb167e05bc4564a14ef8fed70cf057\",\"_SYSTEMD_INVOCATION_ID\":\"3d98d53d32c94d2a89532e1d10825bf5\",\"hostname\":\"master-2\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"info\",\"log_source\":\"node\",\"log_type\":\"infrastructure\",\"message\":\"I0505 23:47:24.008807 2543 kubelet.go:2525] \\\"SyncLoop (probe)\\\" probe=\\\"readiness\\\" status=\\\"ready\\\" pod=\\\"openshift-marketplace/redhat-operators-pn9t8\\\"\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-infrastructure\"},\"sequence\":1746546728210056126},\"systemd\":{\"t\":{\"BOOT_ID\":\"1d807b865e62428b903a6c2ea9d70173\",\"CAP_EFFECTIVE\":\"1ffffffffff\",\"CMDLINE\":\"/usr/bin/kubelet --config=/etc/kubernetes/kubelet.conf --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig --container-runtime-endpoint=/var/run/crio/crio.sock --runtime-cgroups=/system.slice/crio.service --node-labels=node-role.kubernetes.io/control-plane,node-role.kubernetes.io/master,node.openshift.io/os_id=rhcos --node-ip=10.20.177.77 --minimum-container-ttl-duration=6m0s --cloud-provider= --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec --hostname-override= --provider-id= --register-with-taints=node-role.kubernetes.io/master=:NoSchedule --pod-infra-container-image=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:546b3635ab9f651032bf142e243809d64391ee7fcdec9c3d0636218c3dd7aa08 --system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=1Gi --v=2\",\"COMM\":\"kubelet\",\"EXE\":\"/usr/bin/kubelet\",\"GID\":\"0\",\"MACHINE_ID\":\"689138b405714a06a820f2760e64ac76\",\"PID\":\"2543\",\"SELINUX_CONTEXT\":\"system_u:system_r:kubelet_t:s0-s0:c0.c1023\",\"STREAM_ID\":\"1ffb167e05bc4564a14ef8fed70cf057\",\"SYSTEMD_CGROUP\":\"/system.slice/kubelet.service\",\"SYSTEMD_INVOCATION_ID\":\"3d98d53d32c94d2a89532e1d10825bf5\",\"SYSTEMD_SLICE\":\"system.slice\",\"SYSTEMD_UNIT\":\"kubelet.service\",\"TRANSPORT\":\"stdout\",\"UID\":\"0\"},\"u\":{\"SYSLOG_FACILITY\":\"3\",\"SYSLOG_IDENTIFIER\":\"kubenswrapper\"}},\"time\":\"2025-05-05T23:47:24+00:00\"}"},{"ts":"2025-05-06T15:52:16.498777869Z","line":"{\"@timestamp\":\"2025-05-05T23:47:23.938580Z\",\"_RUNTIME_SCOPE\":\"system\",\"_STREAM_ID\":\"1ffb167e05bc4564a14ef8fed70cf057\",\"_SYSTEMD_INVOCATION_ID\":\"3d98d53d32c94d2a89532e1d10825bf5\",\"hostname\":\"master-2\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"info\",\"log_source\":\"node\",\"log_type\":\"infrastructure\",\"message\":\"I0505 23:47:23.938539 2543 kubelet.go:2525] \\\"SyncLoop (probe)\\\" probe=\\\"startup\\\" status=\\\"started\\\" pod=\\\"openshift-marketplace/redhat-operators-pn9t8\\\"\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-infrastructure\"},\"sequence\":1746546728209991834},\"systemd\":{\"t\":{\"BOOT_ID\":\"1d807b865e62428b903a6c2ea9d70173\",\"CAP_EFFECTIVE\":\"1ffffffffff\",\"CMDLINE\":\"/usr/bin/kubelet --config=/etc/kubernetes/kubelet.conf --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig --container-runtime-endpoint=/var/run/crio/crio.sock --runtime-cgroups=/system.slice/crio.service --node-labels=node-role.kubernetes.io/control-plane,node-role.kubernetes.io/master,node.openshift.io/os_id=rhcos --node-ip=10.20.177.77 --minimum-container-ttl-duration=6m0s --cloud-provider= --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec --hostname-override= --provider-id= --register-with-taints=node-role.kubernetes.io/master=:NoSchedule --pod-infra-container-image=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:546b3635ab9f651032bf142e243809d64391ee7fcdec9c3d0636218c3dd7aa08 --system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=1Gi --v=2\",\"COMM\":\"kubelet\",\"EXE\":\"/usr/bin/kubelet\",\"GID\":\"0\",\"MACHINE_ID\":\"689138b405714a06a820f2760e64ac76\",\"PID\":\"2543\",\"SELINUX_CONTEXT\":\"system_u:system_r:kubelet_t:s0-s0:c0.c1023\",\"STREAM_ID\":\"1ffb167e05bc4564a14ef8fed70cf057\",\"SYSTEMD_CGROUP\":\"/system.slice/kubelet.service\",\"SYSTEMD_INVOCATION_ID\":\"3d98d53d32c94d2a89532e1d10825bf5\",\"SYSTEMD_SLICE\":\"system.slice\",\"SYSTEMD_UNIT\":\"kubelet.service\",\"TRANSPORT\":\"stdout\",\"UID\":\"0\"},\"u\":{\"SYSLOG_FACILITY\":\"3\",\"SYSLOG_IDENTIFIER\":\"kubenswrapper\"}},\"time\":\"2025-05-05T23:47:23+00:00\"}"},{"ts":"2025-05-06T15:52:16.498757525Z","line":"{\"@timestamp\":\"2025-05-05T23:47:14.951025Z\",\"_RUNTIME_SCOPE\":\"system\",\"_STREAM_ID\":\"1ffb167e05bc4564a14ef8fed70cf057\",\"_SYSTEMD_INVOCATION_ID\":\"3d98d53d32c94d2a89532e1d10825bf5\",\"hostname\":\"master-2\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"info\",\"log_source\":\"node\",\"log_type\":\"infrastructure\",\"message\":\" \u003e\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"labels\":{\"label\":\"6.0_415-infrastructure\"},\"sequence\":1746546728209927693},\"systemd\":{\"t\":{\"BOOT_ID\":\"1d807b865e62428b903a6c2ea9d70173\",\"CAP_EFFECTIVE\":\"1ffffffffff\",\"CMDLINE\":\"/usr/bin/kubelet --config=/etc/kubernetes/kubelet.conf --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig --container-runtime-endpoint=/var/run/crio/crio.sock --runtime-cgroups=/system.slice/crio.service --node-labels=node-role.kubernetes.io/control-plane,node-role.kubernetes.io/master,node.openshift.io/os_id=rhcos --node-ip=10.20.177.77 --minimum-container-ttl-duration=6m0s --cloud-provider= --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec --hostname-override= --provider-id= --register-with-taints=node-role.kubernetes.io/master=:NoSchedule --pod-infra-container-image=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:546b3635ab9f651032bf142e243809d64391ee7fcdec9c3d0636218c3dd7aa08 --system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=1Gi --v=2\",\"COMM\":\"kubelet\",\"EXE\":\"/usr/bin/kubelet\",\"GID\":\"0\",\"MACHINE_ID\":\"689138b405714a06a820f2760e64ac76\",\"PID\":\"2543\",\"SELINUX_CONTEXT\":\"system_u:system_r:kubelet_t:s0-s0:c0.c1023\",\"STREAM_ID\":\"1ffb167e05bc4564a14ef8fed70cf057\",\"SYSTEMD_CGROUP\":\"/system.slice/kubelet.service\",\"SYSTEMD_INVOCATION_ID\":\"3d98d53d32c94d2a89532e1d10825bf5\",\"SYSTEMD_SLICE\":\"system.slice\",\"SYSTEMD_UNIT\":\"kubelet.service\",\"TRANSPORT\":\"stdout\",\"UID\":\"0\"},\"u\":{\"SYSLOG_FACILITY\":\"3\",\"SYSLOG_IDENTIFIER\":\"kubenswrapper\"}},\"time\":\"2025-05-05T23:47:14+00:00\"}"}]}],"stats":{"summary":{"bytesProcessedPerSecond":640515482,"linesProcessedPerSecond":319087,"totalBytesProcessed":689570214,"totalLinesProcessed":343525,"execTime":1.076586333,"queueTime":0.000103,"subqueries":0,"totalEntriesReturned":100,"splits":1,"shards":0},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0}}},"ingester":{"totalReached":1,"totalChunksMatched":113,"totalBatches":1,"totalLinesSent":100,"store":{"totalChunksRef":1,"totalChunksDownloaded":1,"chunksDownloadTime":1309245,"chunk":{"headChunkBytes":9270866,"headChunkLines":4411,"decompressedBytes":680299348,"decompressedLines":339114,"compressedBytes":48551401,"totalDuplicates":0}}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0}}}} ``` Kafka: Application logs: ``` {"@timestamp":"2025-05-07T02:08:22.300823071Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.76/23\"],\"mac_address\":\"0a:58:0a:83:00:4c\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.76/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.76\"\n ],\n \"mac\": \"0a:58:0a:83:00:4c\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://5ebe6cfeb054b528e3c048d15d7a12e40e474e1e3fb95b417a4478270fddbfb6","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"test-logging","pod_id":"c603b0ef-f979-4a5f-9e69-9ba0d44d3808","pod_ip":"10.131.0.76","pod_name":"centos-logtest-kfr6v","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-application"},"sequence":1746593274773342743}} {"@timestamp":"2025-05-07T02:08:23.302302671Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.76/23\"],\"mac_address\":\"0a:58:0a:83:00:4c\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.76/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.76\"\n ],\n \"mac\": \"0a:58:0a:83:00:4c\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://5ebe6cfeb054b528e3c048d15d7a12e40e474e1e3fb95b417a4478270fddbfb6","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"test-logging","pod_id":"c603b0ef-f979-4a5f-9e69-9ba0d44d3808","pod_ip":"10.131.0.76","pod_name":"centos-logtest-kfr6v","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-application"},"sequence":1746593274773434759}} {"@timestamp":"2025-05-07T02:08:24.303323769Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.76/23\"],\"mac_address\":\"0a:58:0a:83:00:4c\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.76/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.76\"\n ],\n \"mac\": \"0a:58:0a:83:00:4c\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://5ebe6cfeb054b528e3c048d15d7a12e40e474e1e3fb95b417a4478270fddbfb6","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"test-logging","pod_id":"c603b0ef-f979-4a5f-9e69-9ba0d44d3808","pod_ip":"10.131.0.76","pod_name":"centos-logtest-kfr6v","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-application"},"sequence":1746593274773528763}} {"@timestamp":"2025-05-07T02:08:25.305013791Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.76/23\"],\"mac_address\":\"0a:58:0a:83:00:4c\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.76/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.76\"\n ],\n \"mac\": \"0a:58:0a:83:00:4c\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://5ebe6cfeb054b528e3c048d15d7a12e40e474e1e3fb95b417a4478270fddbfb6","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"test-logging","pod_id":"c603b0ef-f979-4a5f-9e69-9ba0d44d3808","pod_ip":"10.131.0.76","pod_name":"centos-logtest-kfr6v","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-application"},"sequence":1746593274773621757}} {"@timestamp":"2025-05-07T02:08:26.306503452Z","hostname":"worker-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.76/23\"],\"mac_address\":\"0a:58:0a:83:00:4c\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.76/23\",\"gateway_ip\":\"10.131.0.1\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.76\"\n ],\n \"mac\": \"0a:58:0a:83:00:4c\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://5ebe6cfeb054b528e3c048d15d7a12e40e474e1e3fb95b417a4478270fddbfb6","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"e2166d8f-cd66-4fec-b7fe-df5234e1507a","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"test-logging","pod_id":"c603b0ef-f979-4a5f-9e69-9ba0d44d3808","pod_ip":"10.131.0.76","pod_name":"centos-logtest-kfr6v","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-application"},"sequence":1746593274773710575}} ``` Audit logs: ``` {"@timestamp":"2025-05-07T04:47:38.510994056Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""},"apiVersion":"audit.k8s.io/v1","auditID":"00a30c64-1837-4b42-972b-00202dc40a08","hostname":"master-2","kind":"Event","level":"Metadata","log_source":"openshiftAPI","log_type":"audit","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-audit"},"sequence":1746593258701219692},"openshift_audit_level":"Metadata","requestReceivedTimestamp":"2025-05-06T23:39:53.937820Z","requestURI":"/openapi/v3","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.129.0.2"],"stage":"ResponseComplete","stageTimestamp":"2025-05-06T23:39:53.938516Z","user":{"groups":["system:authenticated"],"username":"system:aggregator"},"verb":"get"} {"@timestamp":"2025-05-07T04:47:38.511026087Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"console-operator\" of ClusterRole \"console-operator\" to ServiceAccount \"console-operator/openshift-console-operator\""},"apiVersion":"audit.k8s.io/v1","auditID":"9209fced-b923-4f9c-be3c-9f4d8e472d0f","hostname":"master-2","kind":"Event","level":"Metadata","log_source":"openshiftAPI","log_type":"audit","objectRef":{"apiGroup":"oauth.openshift.io","apiVersion":"v1","name":"console","resource":"oauthclients"},"openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-audit"},"sequence":1746593258701259838},"openshift_audit_level":"Metadata","requestReceivedTimestamp":"2025-05-02T06:15:14.261986Z","requestURI":"/apis/oauth.openshift.io/v1/oauthclients/console","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.20.177.77","10.129.0.2"],"stage":"ResponseComplete","stageTimestamp":"2025-05-02T06:15:14.267796Z","user":{"extra":{"authentication.kubernetes.io/pod-name":["console-operator-7fd447cff9-65gkg"],"authentication.kubernetes.io/pod-uid":["0690cc17-2380-4368-8d50-e88697fbfc49"]},"groups":["system:serviceaccounts","system:serviceaccounts:openshift-console-operator","system:authenticated"],"username":"system:serviceaccount:openshift-console-operator:console-operator"},"userAgent":"console/v0.0.0 (linux/ppc64le) kubernetes/$Format","verb":"get"} {"@timestamp":"2025-05-07T04:47:38.511046431Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"prometheus-k8s\" of ClusterRole \"prometheus-k8s\" to ServiceAccount \"prometheus-k8s/openshift-monitoring\""},"apiVersion":"audit.k8s.io/v1","auditID":"992478e3-e498-4896-abf2-d2bd7dcb65d4","hostname":"master-2","kind":"Event","level":"Metadata","log_source":"openshiftAPI","log_type":"audit","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-audit"},"sequence":1746593258701352863},"openshift_audit_level":"Metadata","requestReceivedTimestamp":"2025-05-02T06:15:18.699766Z","requestURI":"/metrics","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.128.2.15"],"stage":"ResponseComplete","stageTimestamp":"2025-05-02T06:15:18.742603Z","user":{"groups":["system:authenticated"],"username":"system:serviceaccount:openshift-monitoring:prometheus-k8s"},"userAgent":"Prometheus/2.48.0","verb":"get"} {"@timestamp":"2025-05-07T04:47:38.511062434Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"prometheus-k8s\" of ClusterRole \"prometheus-k8s\" to ServiceAccount \"prometheus-k8s/openshift-monitoring\""},"apiVersion":"audit.k8s.io/v1","auditID":"198ec2ce-3bc1-4bd6-b1d2-7d304cdeafd9","hostname":"master-2","kind":"Event","level":"Metadata","log_source":"openshiftAPI","log_type":"audit","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-audit"},"sequence":1746593258701383293},"openshift_audit_level":"Metadata","requestReceivedTimestamp":"2025-05-02T06:15:19.795243Z","requestURI":"/metrics","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.131.0.14"],"stage":"ResponseComplete","stageTimestamp":"2025-05-02T06:15:19.845913Z","user":{"groups":["system:authenticated"],"username":"system:serviceaccount:openshift-monitoring:prometheus-k8s"},"userAgent":"Prometheus/2.48.0","verb":"get"} {"@timestamp":"2025-05-07T04:47:38.511079399Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cluster-status-binding\" of ClusterRole \"cluster-status\" to Group \"system:authenticated\""},"apiVersion":"audit.k8s.io/v1","auditID":"bf00efe9-9ef9-4657-a1b6-a34444173c17","hostname":"master-2","kind":"Event","level":"Metadata","log_source":"openshiftAPI","log_type":"audit","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-audit"},"sequence":1746593258701410853},"openshift_audit_level":"Metadata","requestReceivedTimestamp":"2025-05-02T06:15:20.255633Z","requestURI":"/openapi/v2","responseStatus":{"code":304,"metadata":{}},"sourceIPs":["10.130.0.2"],"stage":"ResponseComplete","stageTimestamp":"2025-05-02T06:15:20.259208Z","user":{"groups":["system:authenticated"],"username":"system:aggregator"},"verb":"get"} {"@timestamp":"2025-05-07T04:47:38.511097377Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""},"apiVersion":"audit.k8s.io/v1","auditID":"96fb286f-b63b-45e4-b6a4-e0d694743df0","hostname":"master-2","kind":"Event","level":"Metadata","log_source":"openshiftAPI","log_type":"audit","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-audit"},"sequence":1746593258701439892},"openshift_audit_level":"Metadata","requestReceivedTimestamp":"2025-05-02T06:15:20.406102Z","requestURI":"/openapi/v3","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.130.0.2"],"stage":"ResponseComplete","stageTimestamp":"2025-05-02T06:15:20.409219Z","user":{"groups":["system:authenticated"],"username":"system:aggregator"},"verb":"get"} {"@timestamp":"2025-05-07T04:47:38.511119856Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:openshift:operator:authentication\" of ClusterRole \"cluster-admin\" to ServiceAccount \"authentication-operator/openshift-authentication-operator\""},"apiVersion":"audit.k8s.io/v1","auditID":"59f2a49b-8d77-4165-8305-4be090e62342","hostname":"master-2","kind":"Event","level":"Metadata","log_source":"openshiftAPI","log_type":"audit","objectRef":{"apiGroup":"oauth.openshift.io","apiVersion":"v1","resource":"oauthclients"},"openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-audit"},"sequence":1746593258701478540},"openshift_audit_level":"Metadata","requestReceivedTimestamp":"2025-05-02T06:14:51.155589Z","requestURI":"/apis/oauth.openshift.io/v1/oauthclients?allowWatchBookmarks=true&resourceVersion=36286&timeout=6m55s&timeoutSeconds=415&watch=true","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.129.0.30","10.129.0.2"],"stage":"ResponseComplete","stageTimestamp":"2025-05-02T06:15:25.610755Z","user":{"extra":{"authentication.kubernetes.io/pod-name":["authentication-operator-7b966b4ddd-snsg4"],"authentication.kubernetes.io/pod-uid":["924a078a-b5c3-46e9-9116-b213dea1ad7e"]},"groups":["system:serviceaccounts","system:serviceaccounts:openshift-authentication-operator","system:authenticated"],"username":"system:serviceaccount:openshift-authentication-operator:authentication-operator"},"userAgent":"authentication-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","verb":"watch"} ``` Infrastructure logs: ``` {"@timestamp":"2025-05-07T04:53:53.415903780Z","hostname":"worker-1","kubernetes":{"annotations":{"network.operator.openshift.io/ovnkube-script-lib-hash":"4a77659e2dac3767b552fa56adac97090874874c","networkoperator.openshift.io/cluster-network-cidr":"10.128.0.0/14","networkoperator.openshift.io/hybrid-overlay-status":"disabled","networkoperator.openshift.io/ip-family-mode":"single-stack"},"container_id":"cri-o://3ca0a3364ba292026d4f227a0ebf244f0bfa66c8fbf26c23d0c3e833cdd4ff96","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718","container_name":"ovn-acl-logging","labels":{"app":"ovnkube-node","component":"network","controller-revision-hash":"749c697b97","kubernetes_io_os":"linux","openshift_io_component":"network","ovn-db-pod":"true","pod-template-generation":"2","type":"infra"},"namespace_id":"11ce8fec-c14b-41c2-a43c-e042561435e3","namespace_labels":{"kubernetes_io_metadata_name":"openshift-ovn-kubernetes","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-ovn-kubernetes","pod_id":"0cb2978a-014a-478f-9011-89f892d62a67","pod_ip":"10.20.177.130","pod_name":"ovnkube-node-gr8hg","pod_owner":"DaemonSet/ovnkube-node"},"level":"default","log_source":"container","log_type":"infrastructure","message":"+ true","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-infrastructure"},"sequence":1746593634677281219}} {"@timestamp":"2025-05-07T04:53:53.415903780Z","hostname":"worker-1","kubernetes":{"annotations":{"network.operator.openshift.io/ovnkube-script-lib-hash":"4a77659e2dac3767b552fa56adac97090874874c","networkoperator.openshift.io/cluster-network-cidr":"10.128.0.0/14","networkoperator.openshift.io/hybrid-overlay-status":"disabled","networkoperator.openshift.io/ip-family-mode":"single-stack"},"container_id":"cri-o://3ca0a3364ba292026d4f227a0ebf244f0bfa66c8fbf26c23d0c3e833cdd4ff96","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718","container_name":"ovn-acl-logging","labels":{"app":"ovnkube-node","component":"network","controller-revision-hash":"749c697b97","kubernetes_io_os":"linux","openshift_io_component":"network","ovn-db-pod":"true","pod-template-generation":"2","type":"infra"},"namespace_id":"11ce8fec-c14b-41c2-a43c-e042561435e3","namespace_labels":{"kubernetes_io_metadata_name":"openshift-ovn-kubernetes","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-ovn-kubernetes","pod_id":"0cb2978a-014a-478f-9011-89f892d62a67","pod_ip":"10.20.177.130","pod_name":"ovnkube-node-gr8hg","pod_owner":"DaemonSet/ovnkube-node"},"level":"default","log_source":"container","log_type":"infrastructure","message":"+ '[' -f /var/log/ovn/acl-audit-log.log ']'","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-infrastructure"},"sequence":1746593634677311688}} {"@timestamp":"2025-05-07T04:53:53.416743829Z","hostname":"worker-1","kubernetes":{"annotations":{"network.operator.openshift.io/ovnkube-script-lib-hash":"4a77659e2dac3767b552fa56adac97090874874c","networkoperator.openshift.io/cluster-network-cidr":"10.128.0.0/14","networkoperator.openshift.io/hybrid-overlay-status":"disabled","networkoperator.openshift.io/ip-family-mode":"single-stack"},"container_id":"cri-o://3ca0a3364ba292026d4f227a0ebf244f0bfa66c8fbf26c23d0c3e833cdd4ff96","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718","container_name":"ovn-acl-logging","labels":{"app":"ovnkube-node","component":"network","controller-revision-hash":"749c697b97","kubernetes_io_os":"linux","openshift_io_component":"network","ovn-db-pod":"true","pod-template-generation":"2","type":"infra"},"namespace_id":"11ce8fec-c14b-41c2-a43c-e042561435e3","namespace_labels":{"kubernetes_io_metadata_name":"openshift-ovn-kubernetes","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-ovn-kubernetes","pod_id":"0cb2978a-014a-478f-9011-89f892d62a67","pod_ip":"10.20.177.130","pod_name":"ovnkube-node-gr8hg","pod_owner":"DaemonSet/ovnkube-node"},"level":"default","log_source":"container","log_type":"infrastructure","message":"++ du -b /var/log/ovn/acl-audit-log.log","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-infrastructure"},"sequence":1746593634677337848}} {"@timestamp":"2025-05-07T04:53:53.416832030Z","hostname":"worker-1","kubernetes":{"annotations":{"network.operator.openshift.io/ovnkube-script-lib-hash":"4a77659e2dac3767b552fa56adac97090874874c","networkoperator.openshift.io/cluster-network-cidr":"10.128.0.0/14","networkoperator.openshift.io/hybrid-overlay-status":"disabled","networkoperator.openshift.io/ip-family-mode":"single-stack"},"container_id":"cri-o://3ca0a3364ba292026d4f227a0ebf244f0bfa66c8fbf26c23d0c3e833cdd4ff96","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718","container_name":"ovn-acl-logging","labels":{"app":"ovnkube-node","component":"network","controller-revision-hash":"749c697b97","kubernetes_io_os":"linux","openshift_io_component":"network","ovn-db-pod":"true","pod-template-generation":"2","type":"infra"},"namespace_id":"11ce8fec-c14b-41c2-a43c-e042561435e3","namespace_labels":{"kubernetes_io_metadata_name":"openshift-ovn-kubernetes","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-ovn-kubernetes","pod_id":"0cb2978a-014a-478f-9011-89f892d62a67","pod_ip":"10.20.177.130","pod_name":"ovnkube-node-gr8hg","pod_owner":"DaemonSet/ovnkube-node"},"level":"default","log_source":"container","log_type":"infrastructure","message":"++ tr -s '\\t' ' '","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-infrastructure"},"sequence":1746593634677363134}} {"@timestamp":"2025-05-07T04:53:53.417012937Z","hostname":"worker-1","kubernetes":{"annotations":{"network.operator.openshift.io/ovnkube-script-lib-hash":"4a77659e2dac3767b552fa56adac97090874874c","networkoperator.openshift.io/cluster-network-cidr":"10.128.0.0/14","networkoperator.openshift.io/hybrid-overlay-status":"disabled","networkoperator.openshift.io/ip-family-mode":"single-stack"},"container_id":"cri-o://3ca0a3364ba292026d4f227a0ebf244f0bfa66c8fbf26c23d0c3e833cdd4ff96","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718","container_name":"ovn-acl-logging","labels":{"app":"ovnkube-node","component":"network","controller-revision-hash":"749c697b97","kubernetes_io_os":"linux","openshift_io_component":"network","ovn-db-pod":"true","pod-template-generation":"2","type":"infra"},"namespace_id":"11ce8fec-c14b-41c2-a43c-e042561435e3","namespace_labels":{"kubernetes_io_metadata_name":"openshift-ovn-kubernetes","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-ovn-kubernetes","pod_id":"0cb2978a-014a-478f-9011-89f892d62a67","pod_ip":"10.20.177.130","pod_name":"ovnkube-node-gr8hg","pod_owner":"DaemonSet/ovnkube-node"},"level":"default","log_source":"container","log_type":"infrastructure","message":"++ cut '-d ' -f1","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-infrastructure"},"sequence":1746593634677388112}} {"@timestamp":"2025-05-07T04:53:53.419255231Z","hostname":"worker-1","kubernetes":{"annotations":{"network.operator.openshift.io/ovnkube-script-lib-hash":"4a77659e2dac3767b552fa56adac97090874874c","networkoperator.openshift.io/cluster-network-cidr":"10.128.0.0/14","networkoperator.openshift.io/hybrid-overlay-status":"disabled","networkoperator.openshift.io/ip-family-mode":"single-stack"},"container_id":"cri-o://3ca0a3364ba292026d4f227a0ebf244f0bfa66c8fbf26c23d0c3e833cdd4ff96","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718","container_name":"ovn-acl-logging","labels":{"app":"ovnkube-node","component":"network","controller-revision-hash":"749c697b97","kubernetes_io_os":"linux","openshift_io_component":"network","ovn-db-pod":"true","pod-template-generation":"2","type":"infra"},"namespace_id":"11ce8fec-c14b-41c2-a43c-e042561435e3","namespace_labels":{"kubernetes_io_metadata_name":"openshift-ovn-kubernetes","olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-ovn-kubernetes","pod_id":"0cb2978a-014a-478f-9011-89f892d62a67","pod_ip":"10.20.177.130","pod_name":"ovnkube-node-gr8hg","pod_owner":"DaemonSet/ovnkube-node"},"level":"default","log_source":"container","log_type":"infrastructure","message":"+ file_size=0","openshift":{"cluster_id":"ae9a9f3f-409f-486a-9228-008db9788e0c","labels":{"label":"6.0.8_415-infrastructure"},"sequence":1746593634677417290}} ```