Cluster logging 6.1.5 on OCP 4.17 CSV and Pods: ``` [root@rdr-man-417log-bastion-0 ~]# oc get csv -n openshift-logging NAME DISPLAY VERSION REPLACES PHASE cluster-logging.v6.1.5 Red Hat OpenShift Logging 6.1.5 cluster-logging.v6.1.4 Succeeded loki-operator.v6.1.5 Loki Operator 6.1.5 loki-operator.v6.1.4 Succeeded [root@rdr-man-417log-bastion-0 ~]# oc get pods -n openshift-logging NAME READY STATUS RESTARTS AGE cluster-logging-operator-54495875c6-5zlrt 1/1 Running 0 20m lokistack-cauvp-compactor-0 1/1 Running 0 19m lokistack-cauvp-distributor-69d897b8b-84vvl 1/1 Running 0 19m lokistack-cauvp-gateway-77b4cf84cd-f6vkc 2/2 Running 0 19m lokistack-cauvp-gateway-77b4cf84cd-ggzf2 2/2 Running 0 19m lokistack-cauvp-index-gateway-0 1/1 Running 0 19m lokistack-cauvp-ingester-0 1/1 Running 0 19m lokistack-cauvp-querier-694767cdff-m2wfg 1/1 Running 0 19m lokistack-cauvp-query-frontend-765577645c-rwgnj 1/1 Running 0 19m lokistack-cauvp-ruler-0 1/1 Running 0 19m [root@rdr-man-417log-bastion-0 ~]# ``` Cluster logging: CloudWatch: Application logs: ``` {'timestamp': 1744799103051, 'message': '{"@timestamp":"2025-04-16T08:41:49.048196707Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.210/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:d2\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"169.254.0.5/32\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.210/23\\",\\"gateway_ip\\":\\"10.131.0.1\\",\\"role\\":\\"primary\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.210\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:d2\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://686640542f25368ba3e28cd07b00639208263d95ea1899ee220e194a22dbb922","container_image":"quay.io/powercloud/acmeair-authservice-java:1.0","container_image_id":"quay.io/powercloud/acmeair-authservice-java@sha256:ac57adc499c7d71ba889c90452b0786a7d96de3f83eb5d0db51bc9f373c3cad3","container_iostream":"stdout","container_name":"acmeair-authservice-java","labels":{"name":"acmeair-auth-deployment","pod-template-hash":"8649b5fc8b"},"namespace_id":"13f9d1da-d8b3-4c11-8487-929f8ba8bccc","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"acme-air","pod_id":"337afd93-443a-4394-a17d-f917b0f9f1dd","pod_ip":"10.131.0.210","pod_name":"acmeair-authservice-8649b5fc8b-4cf47","pod_owner":"ReplicaSet/acmeair-authservice-8649b5fc8b"},"level":"default","log_source":"container","log_type":"application","message":"[AUDIT ] CWWKZ0001I: Application acmeair-authservice started in 32.306 seconds.","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799089666569915},"stream_name":"acme-air_acmeair-authservice-8649b5fc8b-4cf47_acmeair-authservice-java"}', 'ingestionTime': 1744799105600} {'timestamp': 1744799103056, 'message': '{"@timestamp":"2025-04-16T08:43:08.131630381Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.213/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:d5\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"169.254.0.5/32\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.213/23\\",\\"gateway_ip\\":\\"10.131.0.1\\",\\"role\\":\\"primary\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.213\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:d5\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://f4541bb456f49222c75388dd8ec10f6e4349ef22f86c3cfb942573912539cb4f","container_image":"ibmcom/icp-mongodb-ppc64le:4.0.12","container_image_id":"docker.io/ibmcom/icp-mongodb-ppc64le@sha256:f54976512a396366c7947073c656e4bb2b33bd3177ae758a23e3630f37bec7d5","container_iostream":"stdout","container_name":"acmeair-booking-db","labels":{"name":"acmeair-booking-db","pod-template-hash":"756c74b547"},"namespace_id":"13f9d1da-d8b3-4c11-8487-929f8ba8bccc","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"acme-air","pod_id":"00f03a3b-5d75-4618-a653-b195b5708a1f","pod_ip":"10.131.0.213","pod_name":"acmeair-booking-db-756c74b547-4vpdt","pod_owner":"ReplicaSet/acmeair-booking-db-756c74b547"},"level":"default","log_source":"container","log_type":"application","message":"2025-04-16T08:43:08.131+0000 I NETWORK [conn3] received client metadata from 10.131.0.218:44428 conn3: { driver: { name: \\"mongo-java-driver|legacy\\", version: \\"3.10.1\\" }, os: { type: \\"Linux\\", name: \\"Linux\\", architecture: \\"ppc64le\\", version: \\"5.14.0-427.62.1.el9_4.ppc64le\\" }, platform: \\"Java/Eclipse OpenJ9/1.8.0_262-b10\\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799089680846375},"stream_name":"acme-air_acmeair-booking-db-756c74b547-4vpdt_acmeair-booking-db"}', 'ingestionTime': 1744799106781} {'timestamp': 1744799103057, 'message': '{"@timestamp":"2025-04-16T08:42:08.750081498Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.218/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:da\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"169.254.0.5/32\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.218/23\\",\\"gateway_ip\\":\\"10.131.0.1\\",\\"role\\":\\"primary\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.218\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:da\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","prometheus.io/port":"9080","prometheus.io/scrape":"true","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://935c2d3a2f3f129c5d2004589d0114caf7777bed5ca552c6b73ee4f50822f233","container_image":"quay.io/powercloud/acmeair-bookingservice-java:1.0","container_image_id":"quay.io/powercloud/acmeair-bookingservice-java@sha256:03c40c5bf1bbc60913651d5113c8d57ad436bb5031667389835bf6b67a8fbb83","container_iostream":"stdout","container_name":"acmeair-bookingservice-java","labels":{"name":"acmeair-booking-deployment","pod-template-hash":"9c9cb69b7"},"namespace_id":"13f9d1da-d8b3-4c11-8487-929f8ba8bccc","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"acme-air","pod_id":"bc960d7b-4d31-449b-a59a-df83c0a88d43","pod_ip":"10.131.0.218","pod_name":"acmeair-bookingservice-9c9cb69b7-25qzn","pod_owner":"ReplicaSet/acmeair-bookingservice-9c9cb69b7"},"level":"warn","log_source":"container","log_type":"application","message":"[WARNING ] No suitable sender found. Using NoopSender, meaning that data will not be sent anywhere!","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799089682442508},"stream_name":"acme-air_acmeair-bookingservice-9c9cb69b7-25qzn_acmeair-bookingservice-java"}', 'ingestionTime': 1744799107309} {'timestamp': 1744799103052, 'message': '{"@timestamp":"2025-04-16T08:43:05.671674817Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.207/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:cf\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"169.254.0.5/32\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.207/23\\",\\"gateway_ip\\":\\"10.131.0.1\\",\\"role\\":\\"primary\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.207\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:cf\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://3753d9ff144c1ef260b3f800696d8a5668cfe96a5f0b484b88ba36ede0d0217a","container_image":"ibmcom/icp-mongodb-ppc64le:4.0.12","container_image_id":"docker.io/ibmcom/icp-mongodb-ppc64le@sha256:f54976512a396366c7947073c656e4bb2b33bd3177ae758a23e3630f37bec7d5","container_iostream":"stdout","container_name":"acmeair-customer-db","labels":{"name":"acmeair-customer-db","pod-template-hash":"8584f45f99"},"namespace_id":"13f9d1da-d8b3-4c11-8487-929f8ba8bccc","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"acme-air","pod_id":"a9c1de7b-4bd7-45a7-a038-eb9d925d186a","pod_ip":"10.131.0.207","pod_name":"acmeair-customer-db-8584f45f99-xlpsz","pod_owner":"ReplicaSet/acmeair-customer-db-8584f45f99"},"level":"default","log_source":"container","log_type":"application","message":"2025-04-16T08:43:05.671+0000 I NETWORK [conn3] received client metadata from 10.131.0.215:34074 conn3: { driver: { name: \\"mongo-java-driver|legacy\\", version: \\"3.10.1\\" }, os: { type: \\"Linux\\", name: \\"Linux\\", architecture: \\"ppc64le\\", version: \\"5.14.0-427.62.1.el9_4.ppc64le\\" }, platform: \\"Java/Eclipse OpenJ9/1.8.0_262-b10\\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799089670608878},"stream_name":"acme-air_acmeair-customer-db-8584f45f99-xlpsz_acmeair-customer-db"}', 'ingestionTime': 1744799106971} ``` Audit logs: ``` {'timestamp': 1744799181068, 'message': '{"@timestamp":"2025-04-16T10:26:13.055+00:00","audit.linux":{"record_id":"29077","type":"PROCTITLE"},"hostname":"master-0","level":"default","log_source":"auditd","log_type":"audit","message":"type=PROCTITLE msg=audit(1744799173.055:29077): proctitle=2F7573722F62696E2F72756E63002D2D73797374656D642D6367726F7570002D2D726F6F743D2F72756E2F72756E6300637265617465002D2D62756E646C65002F72756E2F636F6E7461696E6572732F73746F726167652F6F7665726C61792D636F6E7461696E6572732F663662666162646162323639663364663332643632","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799181056278982},"stream_name":"master-0.auditd"}', 'ingestionTime': 1744799186281} {'timestamp': 1744799193599, 'message': '{"@timestamp":"2025-04-16T10:26:21.368496637Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \\"olm-operator-binding-openshift-operator-lifecycle-manager\\" of ClusterRole \\"system:controller:operator-lifecycle-manager\\" to ServiceAccount \\"olm-operator-serviceaccount/openshift-operator-lifecycle-manager\\""},"apiVersion":"audit.k8s.io/v1","auditID":"83f91091-0e7f-48c0-aa52-027e021d9e19","hostname":"master-0","k8s_audit_level":"Metadata","kind":"Event","level":"Metadata","log_source":"kubeAPI","log_type":"audit","objectRef":{"apiGroup":"apiextensions.k8s.io","apiVersion":"v1","name":"alertingrules.loki.grafana.com","resource":"customresourcedefinitions"},"openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799187790672008},"requestReceivedTimestamp":"2025-04-16T10:19:59.948092Z","requestURI":"/apis/apiextensions.k8s.io/v1/customresourcedefinitions/alertingrules.loki.grafana.com","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.128.1.39"],"stage":"ResponseComplete","stageTimestamp":"2025-04-16T10:19:59.953001Z","stream_name":"master-0.kubeapi","user":{"extra":{"authentication.kubernetes.io/credential-id":["JTI=e438994b-98c2-42c8-ac86-5cc49ab6cb47"],"authentication.kubernetes.io/node-name":["master-0"],"authentication.kubernetes.io/node-uid":["6572ab3b-c5e7-45f3-aa62-3a7701a53f02"],"authentication.kubernetes.io/pod-name":["olm-operator-6b867f88d4-vr69g"],"authentication.kubernetes.io/pod-uid":["aaed4143-5b40-4754-87cc-f00cfe6b0414"]},"groups":["system:serviceaccounts","system:serviceaccounts:openshift-operator-lifecycle-manager","system:authenticated"],"uid":"26e3a5ab-c159-4259-931e-f76f2e185918","username":"system:serviceaccount:openshift-operator-lifecycle-manager:olm-operator-serviceaccount"},"userAgent":"olm/v0.0.0 (linux/ppc64le) kubernetes/$Format","verb":"get"}', 'ingestionTime': 1744799195698} {'timestamp': 1744799190151, 'message': '{"@timestamp":"2025-04-16T10:26:22.814259967Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by RoleBinding \\"cluster-monitoring-operator/openshift-monitoring\\" of ClusterRole \\"cluster-monitoring-operator-namespaced\\" to ServiceAccount \\"cluster-monitoring-operator/openshift-monitoring\\""},"apiVersion":"audit.k8s.io/v1","auditID":"19ffeddd-63a1-47dd-8b2c-d7ec0701659e","hostname":"master-0","kind":"Event","level":"Metadata","log_source":"openshiftAPI","log_type":"audit","objectRef":{"apiGroup":"route.openshift.io","apiVersion":"v1","name":"prometheus-k8s-federate","namespace":"openshift-monitoring","resource":"routes","resourceVersion":"25798"},"openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799184012916760},"openshift_audit_level":"Metadata","requestReceivedTimestamp":"2025-04-16T10:26:22.786376Z","requestURI":"/apis/route.openshift.io/v1/namespaces/openshift-monitoring/routes/prometheus-k8s-federate","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.128.1.35","10.128.0.2"],"stage":"ResponseComplete","stageTimestamp":"2025-04-16T10:26:22.794536Z","stream_name":"master-0.openshiftapi","user":{"extra":{"authentication.kubernetes.io/credential-id":["JTI=4729953b-fc00-4267-9949-358922c349cb"],"authentication.kubernetes.io/node-name":["master-0"],"authentication.kubernetes.io/node-uid":["6572ab3b-c5e7-45f3-aa62-3a7701a53f02"],"authentication.kubernetes.io/pod-name":["cluster-monitoring-operator-676c4d9f66-wtb4x"],"authentication.kubernetes.io/pod-uid":["9965a2ee-8b30-46c4-92c3-bd53b6f1e86b"]},"groups":["system:serviceaccounts","system:serviceaccounts:openshift-monitoring","system:authenticated"],"username":"system:serviceaccount:openshift-monitoring:cluster-monitoring-operator"},"userAgent":"operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","verb":"update"}', 'ingestionTime': 1744799191723} {'timestamp': 1744799185069, 'message': '{"@timestamp":"2025-04-16T07:25:10.226+00:00","audit.linux":{"record_id":"11281","type":"BPF"},"hostname":"master-1","level":"default","log_source":"auditd","log_type":"audit","message":"type=BPF msg=audit(1744788310.226:11281): prog-id=5602 op=UNLOAD","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799180440415861},"stream_name":"master-1.auditd"}', 'ingestionTime': 1744799189240} ``` Infrastructure logs: ``` {'timestamp': 1744799187764, 'message': '{"@timestamp":"2025-04-16T10:26:23.127337Z","SYSLOG_TIMESTAMP":"Apr 16 10:26:23 ","_RUNTIME_SCOPE":"system","_SYSTEMD_INVOCATION_ID":"03402110a1124f0c9daa09e8ba8ef851","hostname":"master-0","level":"notice","log_source":"node","log_type":"infrastructure","message":"ovs|00894|connmgr|INFO|br-int<->unix#6: 146 flow_mods in the 12 s starting 22 s ago (73 adds, 73 deletes)","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799183569690504},"stream_name":"master-0.journal.system","systemd":{"t":{"BOOT_ID":"db82551a8283462e9a0eb154b9f5aed3","CAP_EFFECTIVE":"7c00","CMDLINE":"ovs-vswitchd unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --user openvswitch:hugetlbfs --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach","COMM":"ovs-vswitchd","EXE":"/usr/sbin/ovs-vswitchd","GID":"801","MACHINE_ID":"d75bb6da006044349cf7105cb737eac2","PID":"1658","SELINUX_CONTEXT":"system_u:system_r:openvswitch_t:s0","SYSTEMD_CGROUP":"/system.slice/ovs-vswitchd.service","SYSTEMD_INVOCATION_ID":"03402110a1124f0c9daa09e8ba8ef851","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"ovs-vswitchd.service","TRANSPORT":"syslog","UID":"800"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"ovs-vswitchd"}},"time":"2025-04-16T10:26:23+00:00"}', 'ingestionTime': 1744799189979} {'timestamp': 1744799198460, 'message': '{"@timestamp":"2025-04-16T10:17:51.225633599Z","hostname":"master-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.128.1.10/23\\"],\\"mac_address\\":\\"0a:58:0a:80:01:0a\\",\\"gateway_ips\\":[\\"10.128.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"169.254.0.5/32\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.128.0.1\\"}],\\"ip_address\\":\\"10.128.1.10/23\\",\\"gateway_ip\\":\\"10.128.0.1\\",\\"role\\":\\"primary\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.128.1.10\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:80:01:0a\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/required-scc":"privileged","openshift.io/scc":"privileged","operator.openshift.io/dep-desired.generation":"6","operator.openshift.io/dep-openshift-apiserver.config.configmap":"NIxnGQ==","operator.openshift.io/dep-openshift-apiserver.etcd-client.secret":"LN3K3w==","operator.openshift.io/dep-openshift-apiserver.etcd-serving-ca.configmap":"ho-N9g==","operator.openshift.io/dep-openshift-apiserver.image-import-ca.configmap":"OUOF-g==","operator.openshift.io/dep-openshift-apiserver.trusted-ca-bundle.configmap":"ElMHxA=="},"container_id":"cri-o://7dccfeb00d2db1f8f0f8aa0b0aa0d65e479fab933e3c1b6b367476f9d046549f","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6004190dd7e09f9bd3cc333103f05087e862a9ccd2b6dc89a6b93ea779024eaa","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6004190dd7e09f9bd3cc333103f05087e862a9ccd2b6dc89a6b93ea779024eaa","container_iostream":"stderr","container_name":"openshift-apiserver","labels":{"apiserver":"true","app":"openshift-apiserver-a","openshift-apiserver-anti-affinity":"true","pod-template-hash":"5c7965c67c","revision":"1"},"namespace_id":"d6d941b8-f478-4001-ad87-1adac919cd46","namespace_labels":{"kubernetes_io_metadata_name":"openshift-apiserver","olm_operatorgroup_uid_990bb0f0-956b-4206-a933-bee4b2a24816":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-apiserver","pod_id":"106a5e30-ad5e-4c64-93d7-ce3f4324bd02","pod_ip":"10.128.1.10","pod_name":"apiserver-5c7965c67c-n6vll","pod_owner":"ReplicaSet/apiserver-5c7965c67c"},"level":"warn","log_source":"container","log_type":"infrastructure","message":"W0416 10:17:51.225574 1 logging.go:59] [core] [Channel #5322 SubChannel #5323] grpc: addrConn.createTransport failed to connect to {Addr: \\"10.20.183.166:2379\\", ServerName: \\"10.20.183.166:2379\\", }. Err: connection error: desc = \\"error reading server preface: read tcp 10.128.1.10:51778->10.20.183.166:2379: use of closed network connection\\"","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799198015337308},"stream_name":"master-0.openshift-apiserver_apiserver-5c7965c67c-n6vll_openshift-apiserver"}', 'ingestionTime': 1744799203355} {'timestamp': 1744799198462, 'message': '{"@timestamp":"2025-04-16T08:40:47.126199143Z","hostname":"master-0","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.128.1.10/23\\"],\\"mac_address\\":\\"0a:58:0a:80:01:0a\\",\\"gateway_ips\\":[\\"10.128.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"169.254.0.5/32\\",\\"nextHop\\":\\"10.128.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.128.0.1\\"}],\\"ip_address\\":\\"10.128.1.10/23\\",\\"gateway_ip\\":\\"10.128.0.1\\",\\"role\\":\\"primary\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.128.1.10\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:80:01:0a\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/required-scc":"privileged","openshift.io/scc":"privileged","operator.openshift.io/dep-desired.generation":"6","operator.openshift.io/dep-openshift-apiserver.config.configmap":"NIxnGQ==","operator.openshift.io/dep-openshift-apiserver.etcd-client.secret":"LN3K3w==","operator.openshift.io/dep-openshift-apiserver.etcd-serving-ca.configmap":"ho-N9g==","operator.openshift.io/dep-openshift-apiserver.image-import-ca.configmap":"OUOF-g==","operator.openshift.io/dep-openshift-apiserver.trusted-ca-bundle.configmap":"ElMHxA=="},"container_id":"cri-o://4854aa969f13f00ce4360f21441f0309cfc836da108e1643f77b70fdfe2f9695","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85b0ef8f0817037da66a317319892de0bda3ec718c71f06e19340ea46756ab2d","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85b0ef8f0817037da66a317319892de0bda3ec718c71f06e19340ea46756ab2d","container_iostream":"stderr","container_name":"openshift-apiserver-check-endpoints","labels":{"apiserver":"true","app":"openshift-apiserver-a","openshift-apiserver-anti-affinity":"true","pod-template-hash":"5c7965c67c","revision":"1"},"namespace_id":"d6d941b8-f478-4001-ad87-1adac919cd46","namespace_labels":{"kubernetes_io_metadata_name":"openshift-apiserver","olm_operatorgroup_uid_990bb0f0-956b-4206-a933-bee4b2a24816":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-apiserver","pod_id":"106a5e30-ad5e-4c64-93d7-ce3f4324bd02","pod_ip":"10.128.1.10","pod_name":"apiserver-5c7965c67c-n6vll","pod_owner":"ReplicaSet/apiserver-5c7965c67c"},"level":"info","log_source":"container","log_type":"infrastructure","message":"I0416 08:40:47.126180 1 base_controller.go:110] Starting #1 worker of check-endpoints controller ...","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799198019701221},"stream_name":"master-0.openshift-apiserver_apiserver-5c7965c67c-n6vll_openshift-apiserver-check-endpoints"}', 'ingestionTime': 1744799202633} {'timestamp': 1744799198431, 'message': '{"@timestamp":"2025-04-16T10:25:36.242916631Z","hostname":"master-0","kubernetes":{"annotations":{"kubectl.kubernetes.io/default-container":"cluster-cloud-controller-manager"},"container_id":"cri-o://8a2a8240dd5345758a7f626a695b62cc46c26f71b14fb08f867965ba2ad23b5f","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f1050a63924cce3a22f8e08f04fcfb8d2ecb760b108c02353bb1cebf5b1f7220","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f1050a63924cce3a22f8e08f04fcfb8d2ecb760b108c02353bb1cebf5b1f7220","container_iostream":"stderr","container_name":"cluster-cloud-controller-manager","labels":{"k8s-app":"cloud-manager-operator","pod-template-hash":"69dbbc6b66"},"namespace_id":"3e08c663-1738-46e1-ab4d-12d530127864","namespace_labels":{"kubernetes_io_metadata_name":"openshift-cloud-controller-manager-operator","olm_operatorgroup_uid_990bb0f0-956b-4206-a933-bee4b2a24816":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-cloud-controller-manager-operator","pod_id":"51686353-380a-4d0d-ae2b-d7a8db890885","pod_ip":"10.20.183.26","pod_name":"cluster-cloud-controller-manager-operator-69dbbc6b66-5fsqx","pod_owner":"ReplicaSet/cluster-cloud-controller-manager-operator-69dbbc6b66"},"level":"info","log_source":"container","log_type":"infrastructure","message":"I0416 10:25:36.242862 1 clusteroperator_controller.go:252] Platform does not require an external cloud provider. Skipping...","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799197987669080},"stream_name":"master-0.openshift-cloud-controller-manager-operator_cluster-cloud-controller-manager-operator-69dbbc6b66-5fsqx_cluster-cloud-controller-manager"}', 'ingestionTime': 1744799201837} ``` Elasticsearch : Application logs: ``` {"_index":"application","_type":"_doc","_id":"EqIhPpYBjyXErPejS1kI","_score":1.458417E-4,"_source":{"@timestamp":"2025-04-16T08:53:13.314170909Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.209/23\"],\"mac_address\":\"0a:58:0a:83:00:d1\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.209/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.209\"\n ],\n \"mac\": \"0a:58:0a:83:00:d1\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://b0eb4465735ba3a0b94cc3358d15e3e9d5e21a225d4173f94a0e3e67369e20e2","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_iostream":"stdout","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5ff1c951-ba64-4e3f-be3e-d57653eb21ca","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"test-logging","pod_id":"e2733665-3ee5-4ab6-a6eb-ba974c487acd","pod_ip":"10.131.0.209","pod_name":"centos-logtest-mncq9","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799089569284316}}},{"_index":"application","_type":"_doc","_id":"VaIhPpYBjyXErPejS2aN","_score":1.458417E-4,"_source":{"@timestamp":"2025-04-16T09:49:52.594969933Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.209/23\"],\"mac_address\":\"0a:58:0a:83:00:d1\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.209/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.209\"\n ],\n \"mac\": \"0a:58:0a:83:00:d1\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://b0eb4465735ba3a0b94cc3358d15e3e9d5e21a225d4173f94a0e3e67369e20e2","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_iostream":"stdout","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5ff1c951-ba64-4e3f-be3e-d57653eb21ca","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"test-logging","pod_id":"e2733665-3ee5-4ab6-a6eb-ba974c487acd","pod_ip":"10.131.0.209","pod_name":"centos-logtest-mncq9","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799089539215110}}},{"_index":"application","_type":"_doc","_id":"FKIhPpYBjyXErPejS1kI","_score":1.458417E-4,"_source":{"@timestamp":"2025-04-16T08:53:15.316856462Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.209/23\"],\"mac_address\":\"0a:58:0a:83:00:d1\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.209/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.209\"\n ],\n \"mac\": \"0a:58:0a:83:00:d1\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://b0eb4465735ba3a0b94cc3358d15e3e9d5e21a225d4173f94a0e3e67369e20e2","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_iostream":"stdout","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5ff1c951-ba64-4e3f-be3e-d57653eb21ca","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"test-logging","pod_id":"e2733665-3ee5-4ab6-a6eb-ba974c487acd","pod_ip":"10.131.0.209","pod_name":"centos-logtest-mncq9","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799089569501418}}},{"_index":"application","_type":"_doc","_id":"V6IhPpYBjyXErPejS2aN","_score":1.458417E-4,"_source":{"@timestamp":"2025-04-16T09:49:54.597629804Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.209/23\"],\"mac_address\":\"0a:58:0a:83:00:d1\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.209/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.209\"\n ],\n \"mac\": \"0a:58:0a:83:00:d1\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://b0eb4465735ba3a0b94cc3358d15e3e9d5e21a225d4173f94a0e3e67369e20e2","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_iostream":"stdout","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5ff1c951-ba64-4e3f-be3e-d57653eb21ca","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"test-logging","pod_id":"e2733665-3ee5-4ab6-a6eb-ba974c487acd","pod_ip":"10.131.0.209","pod_name":"centos-logtest-mncq9","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799089539355387}}},{"_index":"application","_type":"_doc","_id":"FqIhPpYBjyXErPejS1kI","_score":1.458417E-4,"_source":{"@timestamp":"2025-04-16T08:53:17.319431026Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.209/23\"],\"mac_address\":\"0a:58:0a:83:00:d1\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.209/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.209\"\n ],\n \"mac\": \"0a:58:0a:83:00:d1\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://b0eb4465735ba3a0b94cc3358d15e3e9d5e21a225d4173f94a0e3e67369e20e2","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_iostream":"stdout","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5ff1c951-ba64-4e3f-be3e-d57653eb21ca","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"test-logging","pod_id":"e2733665-3ee5-4ab6-a6eb-ba974c487acd","pod_ip":"10.131.0.209","pod_name":"centos-logtest-mncq9","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799089569726673}}},{"_index":"application","_type":"_doc","_id":"WaIhPpYBjyXErPejS2aN","_score":1.458417E-4,"_source":{"@timestamp":"2025-04-16T09:49:56.599678609Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.209/23\"],\"mac_address\":\"0a:58:0a:83:00:d1\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.209/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.209\"\n ],\n \"mac\": \"0a:58:0a:83:00:d1\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://b0eb4465735ba3a0b94cc3358d15e3e9d5e21a225d4173f94a0e3e67369e20e2","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_iostream":"stdout","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5ff1c951-ba64-4e3f-be3e-d57653eb21ca","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"test-logging","pod_id":"e2733665-3ee5-4ab6-a6eb-ba974c487acd","pod_ip":"10.131.0.209","pod_name":"centos-logtest-mncq9","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799089539501324}}},{"_index":"application","_type":"_doc","_id":"WqIhPpYBjyXErPejS2aN","_score":1.458417E-4,"_source":{"@timestamp":"2025-04-16T09:49:57.601043131Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.209/23\"],\"mac_address\":\"0a:58:0a:83:00:d1\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.209/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.209\"\n ],\n \"mac\": \"0a:58:0a:83:00:d1\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://b0eb4465735ba3a0b94cc3358d15e3e9d5e21a225d4173f94a0e3e67369e20e2","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_iostream":"stdout","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5ff1c951-ba64-4e3f-be3e-d57653eb21ca","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"test-logging","pod_id":"e2733665-3ee5-4ab6-a6eb-ba974c487acd","pod_ip":"10.131.0.209","pod_name":"centos-logtest-mncq9","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799089539564265}}},{"_index":"application","_type":"_doc","_id":"GaIhPpYBjyXErPejS1kI","_score":1.458417E-4,"_source":{"@timestamp":"2025-04-16T08:53:20.323285939Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.209/23\"],\"mac_address\":\"0a:58:0a:83:00:d1\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.209/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.209\"\n ],\n \"mac\": \"0a:58:0a:83:00:d1\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://b0eb4465735ba3a0b94cc3358d15e3e9d5e21a225d4173f94a0e3e67369e20e2","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_iostream":"stdout","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5ff1c951-ba64-4e3f-be3e-d57653eb21ca","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"test-logging","pod_id":"e2733665-3ee5-4ab6-a6eb-ba974c487acd","pod_ip":"10.131.0.209","pod_name":"centos-logtest-mncq9","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799089570010492}}},{"_index":"application","_type":"_doc","_id":"XKIhPpYBjyXErPejS2aN","_score":1.458417E-4,"_source":{"@timestamp":"2025-04-16T09:49:59.606570187Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.209/23\"],\"mac_address\":\"0a:58:0a:83:00:d1\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.209/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.209\"\n ],\n \"mac\": \"0a:58:0a:83:00:d1\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://b0eb4465735ba3a0b94cc3358d15e3e9d5e21a225d4173f94a0e3e67369e20e2","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_iostream":"stdout","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5ff1c951-ba64-4e3f-be3e-d57653eb21ca","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"test-logging","pod_id":"e2733665-3ee5-4ab6-a6eb-ba974c487acd","pod_ip":"10.131.0.209","pod_name":"centos-logtest-mncq9","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799089539715903}}}]}} ``` Audit logs: ``` {"took":4320,"timed_out":false,"_shards":{"total":2,"successful":2,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":5.734315E-6,"hits":[{"_index":"audit","_type":"_doc","_id":"DaAgPpYBjyXErPej7tDh","_score":5.734315E-6,"_source":{"@timestamp":"2025-04-16T03:10:35.282+00:00","audit.linux":{"record_id":"8966","type":"DAEMON_START"},"hostname":"worker-0","level":"default","log_source":"auditd","log_type":"audit","message":"type=DAEMON_START msg=audit(1744773035.282:8966): op=start ver=3.1.2 format=enriched kernel=5.14.0-427.18.1.el9_4.ppc64le auid=4294967295 pid=2278 uid=0 ses=4294967295 subj=system_u:system_r:auditd_t:s0 res=success\u001dAUID=\"unset\" UID=\"root\"","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799066618918485}}},{"_index":"audit","_type":"_doc","_id":"DqAgPpYBjyXErPej7tDh","_score":5.734315E-6,"_source":{"@timestamp":"2025-04-16T03:10:35.280+00:00","audit.linux":{"record_id":"5","type":"SERVICE_START"},"hostname":"worker-0","level":"default","log_source":"auditd","log_type":"audit","message":"type=SERVICE_START msg=audit(1744773035.280:5): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=rhcos-usrlocal-selinux-fixup comm=\"systemd\" exe=\"/usr/lib/systemd/systemd\" hostname=? addr=? terminal=? res=success'\u001dUID=\"root\" AUID=\"unset\"","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799066619007945}}},{"_index":"audit","_type":"_doc","_id":"D6AgPpYBjyXErPej7tDh","_score":5.734315E-6,"_source":{"@timestamp":"2025-04-16T03:10:35.290+00:00","audit.linux":{"record_id":"6","type":"SERVICE_START"},"hostname":"worker-0","level":"default","log_source":"auditd","log_type":"audit","message":"type=SERVICE_START msg=audit(1744773035.290:6): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=systemd-journal-catalog-update comm=\"systemd\" exe=\"/usr/lib/systemd/systemd\" hostname=? addr=? terminal=? res=success'\u001dUID=\"root\" AUID=\"unset\"","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799066619098359}}},{"_index":"audit","_type":"_doc","_id":"EKAgPpYBjyXErPej7tDh","_score":5.734315E-6,"_source":{"@timestamp":"2025-04-16T03:10:35.300+00:00","audit.linux":{"record_id":"7","type":"SERVICE_START"},"hostname":"worker-0","level":"default","log_source":"auditd","log_type":"audit","message":"type=SERVICE_START msg=audit(1744773035.300:7): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=systemd-update-done comm=\"systemd\" exe=\"/usr/lib/systemd/systemd\" hostname=? addr=? terminal=? res=success'\u001dUID=\"root\" AUID=\"unset\"","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799066619189558}}},{"_index":"audit","_type":"_doc","_id":"EaAgPpYBjyXErPej7tDh","_score":5.734315E-6,"_source":{"@timestamp":"2025-04-16T03:10:35.310+00:00","audit.linux":{"record_id":"8","type":"CONFIG_CHANGE"},"hostname":"worker-0","level":"default","log_source":"auditd","log_type":"audit","message":"type=CONFIG_CHANGE msg=audit(1744773035.310:8): op=set audit_backlog_limit=8192 old=64 auid=4294967295 ses=4294967295 subj=system_u:system_r:unconfined_service_t:s0 res=1\u001dAUID=\"unset\"","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799066619304231}}},{"_index":"audit","_type":"_doc","_id":"EqAgPpYBjyXErPej7tDh","_score":5.734315E-6,"_source":{"@timestamp":"2025-04-16T03:10:35.310+00:00","audit.linux":{"record_id":"8","type":"SYSCALL"},"hostname":"worker-0","level":"default","log_source":"auditd","log_type":"audit","message":"type=SYSCALL msg=audit(1744773035.310:8): arch=c0000015 syscall=335 success=yes exit=60 a0=3 a1=7ffffb428da4 a2=3c a3=0 items=0 ppid=2282 pid=2298 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"auditctl\" exe=\"/usr/sbin/auditctl\" subj=system_u:system_r:unconfined_service_t:s0 key=(null)\u001dARCH=ppc64le SYSCALL=sendto AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\"","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799066619395990}}},{"_index":"audit","_type":"_doc","_id":"E6AgPpYBjyXErPej7tDh","_score":5.734315E-6,"_source":{"@timestamp":"2025-04-16T03:10:35.310+00:00","audit.linux":{"record_id":"8","type":"PROCTITLE"},"hostname":"worker-0","level":"default","log_source":"auditd","log_type":"audit","message":"type=PROCTITLE msg=audit(1744773035.310:8): proctitle=2F7362696E2F617564697463746C002D52002F6574632F61756469742F61756469742E72756C6573","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799066619490347}}},{"_index":"audit","_type":"_doc","_id":"FKAgPpYBjyXErPej7tDh","_score":5.734315E-6,"_source":{"@timestamp":"2025-04-16T03:10:35.310+00:00","audit.linux":{"record_id":"9","type":"CONFIG_CHANGE"},"hostname":"worker-0","level":"default","log_source":"auditd","log_type":"audit","message":"type=CONFIG_CHANGE msg=audit(1744773035.310:9): op=set audit_failure=1 old=1 auid=4294967295 ses=4294967295 subj=system_u:system_r:unconfined_service_t:s0 res=1\u001dAUID=\"unset\"","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799066619593964}}},{"_index":"audit","_type":"_doc","_id":"FaAgPpYBjyXErPej7tDh","_score":5.734315E-6,"_source":{"@timestamp":"2025-04-16T03:10:35.310+00:00","audit.linux":{"record_id":"9","type":"SYSCALL"},"hostname":"worker-0","level":"default","log_source":"auditd","log_type":"audit","message":"type=SYSCALL msg=audit(1744773035.310:9): arch=c0000015 syscall=335 success=yes exit=60 a0=3 a1=7ffffb428da4 a2=3c a3=0 items=0 ppid=2282 pid=2298 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"auditctl\" exe=\"/usr/sbin/auditctl\" subj=system_u:system_r:unconfined_service_t:s0 key=(null)\u001dARCH=ppc64le SYSCALL=sendto AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\"","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799066619693543}}},{"_index":"audit","_type":"_doc","_id":"FqAgPpYBjyXErPej7tDh","_score":5.734315E-6,"_source":{"@timestamp":"2025-04-16T03:10:35.310+00:00","audit.linux":{"record_id":"9","type":"PROCTITLE"},"hostname":"worker-0","level":"default","log_source":"auditd","log_type":"audit","message":"type=PROCTITLE msg=audit(1744773035.310:9): proctitle=2F7362696E2F617564697463746C002D52002F6574632F61756469742F61756469742E72756C6573","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799066619783840}}}]}} ``` Infrastructure logs: ``` {"_index":"infrastructure","_type":"_doc","_id":"BKAgPpYBjyXErPej7tDh","_score":1.2333688E-5,"_source":{"@timestamp":"2025-04-16T07:51:40.252508Z","_RUNTIME_SCOPE":"system","_STREAM_ID":"6cf77aa620d34c2d9a762e32e39cc4a4","_SYSTEMD_INVOCATION_ID":"fc80179ee7c64a39a91079a139539b38","hostname":"worker-0","level":"info","log_source":"node","log_type":"infrastructure","message":"I0416 07:51:40.251778 7847 kuberuntime_container.go:779] \"Killing container with a grace period\" pod=\"openshift-logging/collector-9nmcj\" podUID=\"da67cbb4-77fd-4613-9bde-101cb9a7e87a\" containerName=\"collector\" containerID=\"cri-o://985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83\" gracePeriod=10","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799066539649102},"systemd":{"t":{"BOOT_ID":"eb00a54e66e34856a1a2ef32790f2e3a","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/bin/kubelet --config=/etc/kubernetes/kubelet.conf --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig --container-runtime-endpoint=/var/run/crio/crio.sock --runtime-cgroups=/system.slice/crio.service --node-labels=node-role.kubernetes.io/worker,node.openshift.io/os_id=rhcos --node-ip=10.20.183.129 --minimum-container-ttl-duration=6m0s --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec --cloud-provider= --hostname-override= --provider-id= --pod-infra-container-image=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:040b535f95689b081a0fc0b633496e8a1bccc6789766377550540d0a7b78420c --system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=1Gi --v=2","COMM":"kubelet","EXE":"/usr/bin/kubelet","GID":"0","MACHINE_ID":"0908bdf0029948b09fa09e8dd5296575","PID":"7847","SELINUX_CONTEXT":"system_u:system_r:kubelet_t:s0-s0:c0.c1023","STREAM_ID":"6cf77aa620d34c2d9a762e32e39cc4a4","SYSTEMD_CGROUP":"/system.slice/kubelet.service","SYSTEMD_INVOCATION_ID":"fc80179ee7c64a39a91079a139539b38","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"kubelet.service","TRANSPORT":"stdout","UID":"0"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"kubenswrapper"}},"tag":".journal.system","time":"2025-04-16T07:51:40+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"BaAgPpYBjyXErPej7tDh","_score":1.2333688E-5,"_source":{"@timestamp":"2025-04-16T07:51:40.252632Z","_RUNTIME_SCOPE":"system","_STREAM_ID":"0238636ebbfa4be292934317e16e0813","_SYSTEMD_INVOCATION_ID":"932f3adf40c444939e1a87c404487a2e","hostname":"worker-0","level":"info","log_source":"node","log_type":"infrastructure","message":"time=\"2025-04-16 07:51:40.252357977Z\" level=info msg=\"Stopping container: 985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83 (timeout: 10s)\" id=d45632f9-955c-4d3d-bd87-1dfbec118e77 name=/runtime.v1.RuntimeService/StopContainer","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799066539788924},"systemd":{"t":{"BOOT_ID":"eb00a54e66e34856a1a2ef32790f2e3a","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/bin/crio","COMM":"crio","EXE":"/usr/bin/crio","GID":"0","MACHINE_ID":"0908bdf0029948b09fa09e8dd5296575","PID":"7813","SELINUX_CONTEXT":"system_u:system_r:container_runtime_t:s0","STREAM_ID":"0238636ebbfa4be292934317e16e0813","SYSTEMD_CGROUP":"/system.slice/crio.service","SYSTEMD_INVOCATION_ID":"932f3adf40c444939e1a87c404487a2e","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"crio.service","TRANSPORT":"stdout","UID":"0"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"crio"}},"tag":".journal.system","time":"2025-04-16T07:51:40+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"BqAgPpYBjyXErPej7tDh","_score":1.2333688E-5,"_source":{"@timestamp":"2025-04-16T07:51:50.182264Z","SYSLOG_TIMESTAMP":"Apr 16 07:51:50 ","_RUNTIME_SCOPE":"system","_SYSTEMD_INVOCATION_ID":"7e357f3b456249daafb48e79d881fb26","hostname":"worker-0","level":"notice","log_source":"node","log_type":"infrastructure","message":"ovs|00493|connmgr|INFO|br-int<->unix#6: 10 flow_mods 10 s ago (10 deletes)","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799066539929284},"systemd":{"t":{"BOOT_ID":"eb00a54e66e34856a1a2ef32790f2e3a","CAP_EFFECTIVE":"7c00","CMDLINE":"ovs-vswitchd unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --user openvswitch:hugetlbfs --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach","COMM":"ovs-vswitchd","EXE":"/usr/sbin/ovs-vswitchd","GID":"801","MACHINE_ID":"0908bdf0029948b09fa09e8dd5296575","PID":"1695","SELINUX_CONTEXT":"system_u:system_r:openvswitch_t:s0","SYSTEMD_CGROUP":"/system.slice/ovs-vswitchd.service","SYSTEMD_INVOCATION_ID":"7e357f3b456249daafb48e79d881fb26","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"ovs-vswitchd.service","TRANSPORT":"syslog","UID":"800"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"ovs-vswitchd"}},"tag":".journal.system","time":"2025-04-16T07:51:50+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"B6AgPpYBjyXErPej7tDh","_score":1.2333688E-5,"_source":{"@timestamp":"2025-04-16T07:51:50.262154Z","_RUNTIME_SCOPE":"system","_STREAM_ID":"0238636ebbfa4be292934317e16e0813","_SYSTEMD_INVOCATION_ID":"932f3adf40c444939e1a87c404487a2e","hostname":"worker-0","level":"info","log_source":"node","log_type":"infrastructure","message":"time=\"2025-04-16 07:51:50.262055229Z\" level=warning msg=\"Stopping container 985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83 with stop signal timed out. Killing...\"","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799066540067256},"systemd":{"t":{"BOOT_ID":"eb00a54e66e34856a1a2ef32790f2e3a","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/bin/crio","COMM":"crio","EXE":"/usr/bin/crio","GID":"0","MACHINE_ID":"0908bdf0029948b09fa09e8dd5296575","PID":"7813","SELINUX_CONTEXT":"system_u:system_r:container_runtime_t:s0","STREAM_ID":"0238636ebbfa4be292934317e16e0813","SYSTEMD_CGROUP":"/system.slice/crio.service","SYSTEMD_INVOCATION_ID":"932f3adf40c444939e1a87c404487a2e","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"crio.service","TRANSPORT":"stdout","UID":"0"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"crio"}},"tag":".journal.system","time":"2025-04-16T07:51:50+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"CKAgPpYBjyXErPej7tDh","_score":1.2333688E-5,"_source":{"@timestamp":"2025-04-16T07:51:50.274747Z","SYSLOG_RAW":"<14>Apr 16 07:51:50 conmon: conmon 985b3d7580df70d678c4 : container 297186 exited with status 137 \n","SYSLOG_TIMESTAMP":"Apr 16 07:51:50 ","_RUNTIME_SCOPE":"system","_SYSTEMD_INVOCATION_ID":"421f38e1715f4a789a3952685132edef","hostname":"worker-0","level":"info","log_source":"node","log_type":"infrastructure","message":"conmon 985b3d7580df70d678c4 : container 297186 exited with status 137","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799066540199126},"systemd":{"t":{"BOOT_ID":"eb00a54e66e34856a1a2ef32790f2e3a","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/bin/conmon -b /run/containers/storage/overlay-containers/985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83/userdata -c 985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83 --exit-dir /var/run/crio/exits -l /var/log/pods/openshift-logging_collector-9nmcj_da67cbb4-77fd-4613-9bde-101cb9a7e87a/collector/0.log --log-level info -n k8s_collector_collector-9nmcj_openshift-logging_da67cbb4-77fd-4613-9bde-101cb9a7e87a_0 -P /run/containers/storage/overlay-containers/985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83/userdata/conmon-pidfile -p /run/containers/storage/overlay-containers/985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83/userdata/pidfile --persist-dir /var/lib/containers/storage/overlay-containers/985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83/userdata -r /usr/bin/runc --runtime-arg --root=/run/runc --socket-dir-path /var/run/crio --syslog -u 985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83 -s","COMM":"conmon","EXE":"/usr/bin/conmon","GID":"0","MACHINE_ID":"0908bdf0029948b09fa09e8dd5296575","PID":"297170","SELINUX_CONTEXT":"system_u:system_r:container_runtime_t:s0","SYSTEMD_CGROUP":"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podda67cbb4_77fd_4613_9bde_101cb9a7e87a.slice/crio-conmon-985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83.scope","SYSTEMD_INVOCATION_ID":"421f38e1715f4a789a3952685132edef","SYSTEMD_SLICE":"kubepods-burstable-podda67cbb4_77fd_4613_9bde_101cb9a7e87a.slice","SYSTEMD_UNIT":"crio-conmon-985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83.scope","TRANSPORT":"syslog","UID":"0"},"u":{"SYSLOG_FACILITY":"1","SYSLOG_IDENTIFIER":"conmon"}},"tag":".journal.system","time":"2025-04-16T07:51:50+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"CaAgPpYBjyXErPej7tDh","_score":1.2333688E-5,"_source":{"@timestamp":"2025-04-16T07:51:50.284975Z","INVOCATION_ID":"49df330944e94f62a35ccf46156f28a1","TID":"1","_RUNTIME_SCOPE":"system","hostname":"worker-0","level":"info","log_source":"node","log_type":"infrastructure","message":"var-lib-containers-storage-overlay-22786bac0d853995728af3101f30a511c46db8e985cad63440a293801e7a9db8-merged.mount: Deactivated successfully.","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799066540345208},"systemd":{"t":{"BOOT_ID":"eb00a54e66e34856a1a2ef32790f2e3a","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/lib/systemd/systemd --switched-root --system --deserialize 28","COMM":"systemd","EXE":"/usr/lib/systemd/systemd","GID":"0","MACHINE_ID":"0908bdf0029948b09fa09e8dd5296575","PID":"1","SELINUX_CONTEXT":"system_u:system_r:init_t:s0","SYSTEMD_CGROUP":"/init.scope","SYSTEMD_SLICE":"-.slice","SYSTEMD_UNIT":"init.scope","TRANSPORT":"journal","UID":"0"},"u":{"CODE_FILE":"src/core/unit.c","CODE_FUNCTION":"unit_log_success","CODE_LINE":"5650","MESSAGE_ID":"7ad2d189f7e94e70a38c781354912448","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"systemd","UNIT":"var-lib-containers-storage-overlay-22786bac0d853995728af3101f30a511c46db8e985cad63440a293801e7a9db8-merged.mount"}},"tag":".journal.system","time":"2025-04-16T07:51:50+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"CqAgPpYBjyXErPej7tDh","_score":1.2333688E-5,"_source":{"@timestamp":"2025-04-16T07:51:50.287231Z","INVOCATION_ID":"421f38e1715f4a789a3952685132edef","TID":"1","_RUNTIME_SCOPE":"system","hostname":"worker-0","level":"info","log_source":"node","log_type":"infrastructure","message":"crio-conmon-985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83.scope: Deactivated successfully.","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799066540508090},"systemd":{"t":{"BOOT_ID":"eb00a54e66e34856a1a2ef32790f2e3a","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/lib/systemd/systemd --switched-root --system --deserialize 28","COMM":"systemd","EXE":"/usr/lib/systemd/systemd","GID":"0","MACHINE_ID":"0908bdf0029948b09fa09e8dd5296575","PID":"1","SELINUX_CONTEXT":"system_u:system_r:init_t:s0","SYSTEMD_CGROUP":"/init.scope","SYSTEMD_SLICE":"-.slice","SYSTEMD_UNIT":"init.scope","TRANSPORT":"journal","UID":"0"},"u":{"CODE_FILE":"src/core/unit.c","CODE_FUNCTION":"unit_log_success","CODE_LINE":"5650","MESSAGE_ID":"7ad2d189f7e94e70a38c781354912448","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"systemd","UNIT":"crio-conmon-985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83.scope"}},"tag":".journal.system","time":"2025-04-16T07:51:50+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"C6AgPpYBjyXErPej7tDh","_score":1.2333688E-5,"_source":{"@timestamp":"2025-04-16T07:51:50.288070Z","INVOCATION_ID":"51191a47a2854e0fa0d8cfed84829ee0","TID":"1","_RUNTIME_SCOPE":"system","hostname":"worker-0","level":"info","log_source":"node","log_type":"infrastructure","message":"crio-985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83.scope: Deactivated successfully.","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799066540666909},"systemd":{"t":{"BOOT_ID":"eb00a54e66e34856a1a2ef32790f2e3a","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/lib/systemd/systemd --switched-root --system --deserialize 28","COMM":"systemd","EXE":"/usr/lib/systemd/systemd","GID":"0","MACHINE_ID":"0908bdf0029948b09fa09e8dd5296575","PID":"1","SELINUX_CONTEXT":"system_u:system_r:init_t:s0","SYSTEMD_CGROUP":"/init.scope","SYSTEMD_SLICE":"-.slice","SYSTEMD_UNIT":"init.scope","TRANSPORT":"journal","UID":"0"},"u":{"CODE_FILE":"src/core/unit.c","CODE_FUNCTION":"unit_log_success","CODE_LINE":"5650","MESSAGE_ID":"7ad2d189f7e94e70a38c781354912448","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"systemd","UNIT":"crio-985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83.scope"}},"tag":".journal.system","time":"2025-04-16T07:51:50+00:00"}},{"_index":"infrastructure","_type":"_doc","_id":"DKAgPpYBjyXErPej7tDh","_score":1.2333688E-5,"_source":{"@timestamp":"2025-04-16T07:51:50.288375Z","_RUNTIME_SCOPE":"system","_STREAM_ID":"0238636ebbfa4be292934317e16e0813","_SYSTEMD_INVOCATION_ID":"932f3adf40c444939e1a87c404487a2e","hostname":"worker-0","level":"info","log_source":"node","log_type":"infrastructure","message":"time=\"2025-04-16 07:51:50.288325847Z\" level=info msg=\"Stopped container 985b3d7580df70d678c42d2bd349e6a0e46286fda54989e3376b97b81ac28d83: openshift-logging/collector-9nmcj/collector\" id=d45632f9-955c-4d3d-bd87-1dfbec118e77 name=/runtime.v1.RuntimeService/StopContainer","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799066540823664},"systemd":{"t":{"BOOT_ID":"eb00a54e66e34856a1a2ef32790f2e3a","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/bin/crio","COMM":"crio","EXE":"/usr/bin/crio","GID":"0","MACHINE_ID":"0908bdf0029948b09fa09e8dd5296575","PID":"7813","SELINUX_CONTEXT":"system_u:system_r:container_runtime_t:s0","STREAM_ID":"0238636ebbfa4be292934317e16e0813","SYSTEMD_CGROUP":"/system.slice/crio.service","SYSTEMD_INVOCATION_ID":"932f3adf40c444939e1a87c404487a2e","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"crio.service","TRANSPORT":"stdout","UID":"0"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"crio"}},"tag":".journal.system","time":"2025-04-16T07:51:50+00:00"}}]}} ``` Kafka: Application logs: ``` {"@timestamp":"2025-04-16T10:29:24.681332264Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.209/23\"],\"mac_address\":\"0a:58:0a:83:00:d1\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.209/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.209\"\n ],\n \"mac\": \"0a:58:0a:83:00:d1\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://b0eb4465735ba3a0b94cc3358d15e3e9d5e21a225d4173f94a0e3e67369e20e2","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_iostream":"stdout","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5ff1c951-ba64-4e3f-be3e-d57653eb21ca","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"test-logging","pod_id":"e2733665-3ee5-4ab6-a6eb-ba974c487acd","pod_ip":"10.131.0.209","pod_name":"centos-logtest-mncq9","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799365067635371}} {"@timestamp":"2025-04-16T10:29:25.681800967Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.209/23\"],\"mac_address\":\"0a:58:0a:83:00:d1\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.209/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.209\"\n ],\n \"mac\": \"0a:58:0a:83:00:d1\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://b0eb4465735ba3a0b94cc3358d15e3e9d5e21a225d4173f94a0e3e67369e20e2","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_iostream":"stdout","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5ff1c951-ba64-4e3f-be3e-d57653eb21ca","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"test-logging","pod_id":"e2733665-3ee5-4ab6-a6eb-ba974c487acd","pod_ip":"10.131.0.209","pod_name":"centos-logtest-mncq9","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799366099674442}} {"@timestamp":"2025-04-16T10:29:26.683166645Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.209/23\"],\"mac_address\":\"0a:58:0a:83:00:d1\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.209/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.209\"\n ],\n \"mac\": \"0a:58:0a:83:00:d1\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://b0eb4465735ba3a0b94cc3358d15e3e9d5e21a225d4173f94a0e3e67369e20e2","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_iostream":"stdout","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5ff1c951-ba64-4e3f-be3e-d57653eb21ca","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"test-logging","pod_id":"e2733665-3ee5-4ab6-a6eb-ba974c487acd","pod_ip":"10.131.0.209","pod_name":"centos-logtest-mncq9","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799367133622140}} {"@timestamp":"2025-04-16T10:29:27.684468092Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.209/23\"],\"mac_address\":\"0a:58:0a:83:00:d1\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.209/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.209\"\n ],\n \"mac\": \"0a:58:0a:83:00:d1\",\n \"default\": true,\n \"dns\": {}\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://b0eb4465735ba3a0b94cc3358d15e3e9d5e21a225d4173f94a0e3e67369e20e2","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","container_iostream":"stdout","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5ff1c951-ba64-4e3f-be3e-d57653eb21ca","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"latest","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"latest"},"namespace_name":"test-logging","pod_id":"e2733665-3ee5-4ab6-a6eb-ba974c487acd","pod_ip":"10.131.0.209","pod_name":"centos-logtest-mncq9","pod_owner":"ReplicationController/centos-logtest"},"level":"default","log_source":"container","log_type":"application","message":"{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-application"},"sequence":1744799368165845887}} ``` Audit logs: ``` {"@timestamp":"2025-04-16T10:30:01.325+00:00","audit.linux":{"record_id":"26641","type":"BPF"},"hostname":"worker-1","level":"default","log_source":"auditd","log_type":"audit","message":"type=BPF msg=audit(1744799401.325:26641): prog-id=13328 op=LOAD","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799411500046522}} {"@timestamp":"2025-04-16T10:30:01.325+00:00","audit.linux":{"record_id":"26642","type":"BPF"},"hostname":"worker-1","level":"default","log_source":"auditd","log_type":"audit","message":"type=BPF msg=audit(1744799401.325:26642): prog-id=13329 op=LOAD","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799411500094938}} {"@timestamp":"2025-04-16T10:30:01.325+00:00","audit.linux":{"record_id":"26642","type":"SYSCALL"},"hostname":"worker-1","level":"default","log_source":"auditd","log_type":"audit","message":"type=SYSCALL msg=audit(1744799401.325:26642): arch=c0000015 syscall=361 success=yes exit=14 a0=5 a1=c0001355c8 a2=78 a3=0 items=0 ppid=509133 pid=509134 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"runc\" exe=\"/usr/bin/runc\" subj=system_u:system_r:container_runtime_t:s0 key=(null)\u001dARCH=ppc64le SYSCALL=bpf AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\"","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799411500137320}} {"@timestamp":"2025-04-16T10:30:01.325+00:00","audit.linux":{"record_id":"26642","type":"PROCTITLE"},"hostname":"worker-1","level":"default","log_source":"auditd","log_type":"audit","message":"type=PROCTITLE msg=audit(1744799401.325:26642): proctitle=2F7573722F62696E2F72756E63002D2D73797374656D642D6367726F7570002D2D726F6F743D2F72756E2F72756E6300637265617465002D2D62756E646C65002F72756E2F636F6E7461696E6572732F73746F726167652F6F7665726C61792D636F6E7461696E6572732F363663366631343137653038333861376363333836","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799411500176660}} {"@timestamp":"2025-04-16T10:30:01.325+00:00","audit.linux":{"record_id":"26643","type":"BPF"},"hostname":"worker-1","level":"default","log_source":"auditd","log_type":"audit","message":"type=BPF msg=audit(1744799401.325:26643): prog-id=13330 op=LOAD","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799411500215990}} {"@timestamp":"2025-04-16T10:30:01.325+00:00","audit.linux":{"record_id":"26643","type":"SYSCALL"},"hostname":"worker-1","level":"default","log_source":"auditd","log_type":"audit","message":"type=SYSCALL msg=audit(1744799401.325:26643): arch=c0000015 syscall=361 success=yes exit=16 a0=5 a1=c000135310 a2=78 a3=0 items=0 ppid=509133 pid=509134 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"runc\" exe=\"/usr/bin/runc\" subj=system_u:system_r:container_runtime_t:s0 key=(null)\u001dARCH=ppc64le SYSCALL=bpf AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\"","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799411500255875}} {"@timestamp":"2025-04-16T10:30:01.325+00:00","audit.linux":{"record_id":"26643","type":"PROCTITLE"},"hostname":"worker-1","level":"default","log_source":"auditd","log_type":"audit","message":"type=PROCTITLE msg=audit(1744799401.325:26643): proctitle=2F7573722F62696E2F72756E63002D2D73797374656D642D6367726F7570002D2D726F6F743D2F72756E2F72756E6300637265617465002D2D62756E646C65002F72756E2F636F6E7461696E6572732F73746F726167652F6F7665726C61792D636F6E7461696E6572732F363663366631343137653038333861376363333836","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799411500293892}} {"@timestamp":"2025-04-16T10:30:01.325+00:00","audit.linux":{"record_id":"26644","type":"BPF"},"hostname":"worker-1","level":"default","log_source":"auditd","log_type":"audit","message":"type=BPF msg=audit(1744799401.325:26644): prog-id=13330 op=UNLOAD","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799411500333312}} {"@timestamp":"2025-04-16T10:30:01.325+00:00","audit.linux":{"record_id":"26645","type":"BPF"},"hostname":"worker-1","level":"default","log_source":"auditd","log_type":"audit","message":"type=BPF msg=audit(1744799401.325:26645): prog-id=13329 op=UNLOAD","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799411500370159}} {"@timestamp":"2025-04-16T10:30:01.325+00:00","audit.linux":{"record_id":"26646","type":"BPF"},"hostname":"worker-1","level":"default","log_source":"auditd","log_type":"audit","message":"type=BPF msg=audit(1744799401.325:26646): prog-id=13331 op=LOAD","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-audit"},"sequence":1744799411500408995}} ``` Infrastructure logs: ``` {"@timestamp":"2025-04-16T10:30:15.822893743Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.205/23\"],\"mac_address\":\"0a:58:0a:83:00:cd\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.205/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.205\"\n ],\n \"mac\": \"0a:58:0a:83:00:cd\",\n \"default\": true,\n \"dns\": {}\n}]","kubectl.kubernetes.io/default-container":"prometheus","openshift.io/required-scc":"nonroot","openshift.io/scc":"nonroot"},"container_id":"cri-o://4a25c31f51587cb19c9207f3e8dc2e6910eb78ddd991a410814220cc4bf18c9f","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3d5f80b630407f94f22016e0e05a301543f4283f375d11aa757987772994e055","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3d5f80b630407f94f22016e0e05a301543f4283f375d11aa757987772994e055","container_iostream":"stderr","container_name":"kube-rbac-proxy-web","labels":{"app_kubernetes_io_component":"prometheus","app_kubernetes_io_instance":"k8s","app_kubernetes_io_managed-by":"prometheus-operator","app_kubernetes_io_name":"prometheus","app_kubernetes_io_part-of":"openshift-monitoring","app_kubernetes_io_version":"2.53.1","apps_kubernetes_io_pod-index":"1","controller-revision-hash":"prometheus-k8s-7dc44c7b9","operator_prometheus_io_name":"k8s","operator_prometheus_io_shard":"0","prometheus":"k8s","statefulset_kubernetes_io_pod-name":"prometheus-k8s-1"},"namespace_id":"48c05d13-6a95-4b97-99a9-117782dc44bf","namespace_labels":{"kubernetes_io_metadata_name":"openshift-monitoring","name":"openshift-monitoring","network_openshift_io_policy-group":"monitoring","olm_operatorgroup_uid_990bb0f0-956b-4206-a933-bee4b2a24816":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-monitoring","pod_id":"f01c133b-a46e-4096-b7dc-b4750294b781","pod_ip":"10.131.0.205","pod_name":"prometheus-k8s-1","pod_owner":"StatefulSet/prometheus-k8s"},"level":"info","log_source":"container","log_type":"infrastructure","message":"I0416 10:30:15.822847 1 log.go:245] http: TLS handshake error from 10.128.2.2:35282: write tcp 10.131.0.205:9091->10.128.2.2:35282: write: connection reset by peer","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799416074914075}} {"@timestamp":"2025-04-16T10:30:19.763097287Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.204/23\"],\"mac_address\":\"0a:58:0a:83:00:cc\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.204/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.204\"\n ],\n \"mac\": \"0a:58:0a:83:00:cc\",\n \"default\": true,\n \"dns\": {}\n}]","kubectl.kubernetes.io/default-container":"alertmanager","openshift.io/required-scc":"nonroot","openshift.io/scc":"nonroot"},"container_id":"cri-o://909397d78c7a119b9b6f1426293239a4c9a74b47544bbeb5edec4b1b17c650f8","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3d5f80b630407f94f22016e0e05a301543f4283f375d11aa757987772994e055","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3d5f80b630407f94f22016e0e05a301543f4283f375d11aa757987772994e055","container_iostream":"stderr","container_name":"kube-rbac-proxy-web","labels":{"alertmanager":"main","app_kubernetes_io_component":"alert-router","app_kubernetes_io_instance":"main","app_kubernetes_io_managed-by":"prometheus-operator","app_kubernetes_io_name":"alertmanager","app_kubernetes_io_part-of":"openshift-monitoring","app_kubernetes_io_version":"0.27.0","apps_kubernetes_io_pod-index":"0","controller-revision-hash":"alertmanager-main-5985db4595","statefulset_kubernetes_io_pod-name":"alertmanager-main-0"},"namespace_id":"48c05d13-6a95-4b97-99a9-117782dc44bf","namespace_labels":{"kubernetes_io_metadata_name":"openshift-monitoring","name":"openshift-monitoring","network_openshift_io_policy-group":"monitoring","olm_operatorgroup_uid_990bb0f0-956b-4206-a933-bee4b2a24816":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-monitoring","pod_id":"a9b0593a-bdf5-4b2f-b500-4958b0d03a54","pod_ip":"10.131.0.204","pod_name":"alertmanager-main-0","pod_owner":"StatefulSet/alertmanager-main"},"level":"info","log_source":"container","log_type":"infrastructure","message":"I0416 10:30:19.763060 1 log.go:245] http: TLS handshake error from 10.128.2.2:49314: write tcp 10.131.0.204:9095->10.128.2.2:49314: write: connection reset by peer","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799420181566062}} {"@timestamp":"2025-04-16T10:30:20.829669100Z","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\"default\":{\"ip_addresses\":[\"10.131.0.205/23\"],\"mac_address\":\"0a:58:0a:83:00:cd\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.0.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.205/23\",\"gateway_ip\":\"10.131.0.1\",\"role\":\"primary\"}}","k8s.v1.cni.cncf.io/network-status":"[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.205\"\n ],\n \"mac\": \"0a:58:0a:83:00:cd\",\n \"default\": true,\n \"dns\": {}\n}]","kubectl.kubernetes.io/default-container":"prometheus","openshift.io/required-scc":"nonroot","openshift.io/scc":"nonroot"},"container_id":"cri-o://4a25c31f51587cb19c9207f3e8dc2e6910eb78ddd991a410814220cc4bf18c9f","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3d5f80b630407f94f22016e0e05a301543f4283f375d11aa757987772994e055","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3d5f80b630407f94f22016e0e05a301543f4283f375d11aa757987772994e055","container_iostream":"stderr","container_name":"kube-rbac-proxy-web","labels":{"app_kubernetes_io_component":"prometheus","app_kubernetes_io_instance":"k8s","app_kubernetes_io_managed-by":"prometheus-operator","app_kubernetes_io_name":"prometheus","app_kubernetes_io_part-of":"openshift-monitoring","app_kubernetes_io_version":"2.53.1","apps_kubernetes_io_pod-index":"1","controller-revision-hash":"prometheus-k8s-7dc44c7b9","operator_prometheus_io_name":"k8s","operator_prometheus_io_shard":"0","prometheus":"k8s","statefulset_kubernetes_io_pod-name":"prometheus-k8s-1"},"namespace_id":"48c05d13-6a95-4b97-99a9-117782dc44bf","namespace_labels":{"kubernetes_io_metadata_name":"openshift-monitoring","name":"openshift-monitoring","network_openshift_io_policy-group":"monitoring","olm_operatorgroup_uid_990bb0f0-956b-4206-a933-bee4b2a24816":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-monitoring","pod_id":"f01c133b-a46e-4096-b7dc-b4750294b781","pod_ip":"10.131.0.205","pod_name":"prometheus-k8s-1","pod_owner":"StatefulSet/prometheus-k8s"},"level":"info","log_source":"container","log_type":"infrastructure","message":"I0416 10:30:20.829598 1 log.go:245] http: TLS handshake error from 10.128.2.2:35294: write tcp 10.131.0.205:9091->10.128.2.2:35294: write: connection reset by peer","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799421214070177}} {"@timestamp":"2025-04-16T10:30:24.335258Z","_RUNTIME_SCOPE":"system","_STREAM_ID":"c9b351ed9912405b86f2727033420ab8","_SYSTEMD_INVOCATION_ID":"ec5350961f6241ce909a70b77b219158","hostname":"worker-1","level":"info","log_source":"node","log_type":"infrastructure","message":"I0416 10:30:24.335234 7848 scope.go:117] \"RemoveContainer\" containerID=\"cfd99995e11055c42078cb7692d427d8d46b1d5266100f683bedcbc4e9ba5e4c\"","openshift":{"cluster_id":"d0057771-8074-494e-aee9-f33601c7f37c","labels":{"label":"6.1_417-infrastructure"},"sequence":1744799424662389490},"systemd":{"t":{"BOOT_ID":"3a23db3d02974b1688f279fd31a0b0d2","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/bin/kubelet --config=/etc/kubernetes/kubelet.conf --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig --container-runtime-endpoint=/var/run/crio/crio.sock --runtime-cgroups=/system.slice/crio.service --node-labels=node-role.kubernetes.io/worker,node.openshift.io/os_id=rhcos --node-ip=10.20.183.245 --minimum-container-ttl-duration=6m0s --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec --cloud-provider= --hostname-override= --provider-id= --pod-infra-container-image=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:040b535f95689b081a0fc0b633496e8a1bccc6789766377550540d0a7b78420c --system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=1Gi --v=2","COMM":"kubelet","EXE":"/usr/bin/kubelet","GID":"0","MACHINE_ID":"962d482ad8cc44c38ec3459a8b7f0ea0","PID":"7848","SELINUX_CONTEXT":"system_u:system_r:kubelet_t:s0-s0:c0.c1023","STREAM_ID":"c9b351ed9912405b86f2727033420ab8","SYSTEMD_CGROUP":"/system.slice/kubelet.service","SYSTEMD_INVOCATION_ID":"ec5350961f6241ce909a70b77b219158","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"kubelet.service","TRANSPORT":"stdout","UID":"0"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"kubenswrapper"}},"tag":".journal.system","time":"2025-04-16T10:30:24+00:00"} ``` Loki: Application logs: ``` {\"@timestamp\":\"2025-04-16T08:40:55.492927122Z\",\"hostname\":\"worker-1\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.218/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:da\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.218/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.218\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:da\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"prometheus.io/port\":\"9080\",\"prometheus.io/scrape\":\"true\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://935c2d3a2f3f129c5d2004589d0114caf7777bed5ca552c6b73ee4f50822f233\",\"container_image\":\"quay.io/powercloud/acmeair-bookingservice-java:1.0\",\"container_image_id\":\"quay.io/powercloud/acmeair-bookingservice-java@sha256:03c40c5bf1bbc60913651d5113c8d57ad436bb5031667389835bf6b67a8fbb83\",\"container_iostream\":\"stdout\",\"container_name\":\"acmeair-bookingservice-java\",\"labels\":{\"name\":\"acmeair-booking-deployment\",\"pod-template-hash\":\"9c9cb69b7\"},\"namespace_id\":\"13f9d1da-d8b3-4c11-8487-929f8ba8bccc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"acme-air\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"acme-air\",\"pod_id\":\"bc960d7b-4d31-449b-a59a-df83c0a88d43\",\"pod_ip\":\"10.131.0.218\",\"pod_name\":\"acmeair-bookingservice-9c9cb69b7-25qzn\",\"pod_owner\":\"ReplicaSet/acmeair-bookingservice-9c9cb69b7\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-application\"},\"sequence\":1744799089680908152}}"}]},{"labels":"{k8s_container_name=\"acmeair-booking-db\", k8s_namespace_name=\"acme-air\", k8s_node_name=\"worker-1\", k8s_pod_name=\"acmeair-booking-db-756c74b547-4vpdt\", kubernetes_container_name=\"acmeair-booking-db\", kubernetes_host=\"worker-1\", kubernetes_namespace_name=\"acme-air\", kubernetes_pod_name=\"acmeair-booking-db-756c74b547-4vpdt\", log_type=\"application\", openshift_log_type=\"application\"}","entries":[{"ts":"2025-04-16T10:24:51.700408233Z","line":"{\"@timestamp\":\"2025-04-16T08:43:08.131630381Z\",\"hostname\":\"worker-1\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.213/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:d5\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.213/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.213\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:d5\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://f4541bb456f49222c75388dd8ec10f6e4349ef22f86c3cfb942573912539cb4f\",\"container_image\":\"ibmcom/icp-mongodb-ppc64le:4.0.12\",\"container_image_id\":\"docker.io/ibmcom/icp-mongodb-ppc64le@sha256:f54976512a396366c7947073c656e4bb2b33bd3177ae758a23e3630f37bec7d5\",\"container_iostream\":\"stdout\",\"container_name\":\"acmeair-booking-db\",\"labels\":{\"name\":\"acmeair-booking-db\",\"pod-template-hash\":\"756c74b547\"},\"namespace_id\":\"13f9d1da-d8b3-4c11-8487-929f8ba8bccc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"acme-air\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"acme-air\",\"pod_id\":\"00f03a3b-5d75-4618-a653-b195b5708a1f\",\"pod_ip\":\"10.131.0.213\",\"pod_name\":\"acmeair-booking-db-756c74b547-4vpdt\",\"pod_owner\":\"ReplicaSet/acmeair-booking-db-756c74b547\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"2025-04-16T08:43:08.131+0000 I NETWORK [conn3] received client metadata from 10.131.0.218:44428 conn3: { driver: { name: \\\"mongo-java-driver|legacy\\\", version: \\\"3.10.1\\\" }, os: { type: \\\"Linux\\\", name: \\\"Linux\\\", architecture: \\\"ppc64le\\\", version: \\\"5.14.0-427.62.1.el9_4.ppc64le\\\" }, platform: \\\"Java/Eclipse OpenJ9/1.8.0_262-b10\\\" }\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-application\"},\"sequence\":1744799089680846375}}"},{"ts":"2025-04-16T10:24:51.700364059Z","line":"{\"@timestamp\":\"2025-04-16T08:43:08.130955324Z\",\"hostname\":\"worker-1\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.213/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:d5\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.213/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.213\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:d5\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://f4541bb456f49222c75388dd8ec10f6e4349ef22f86c3cfb942573912539cb4f\",\"container_image\":\"ibmcom/icp-mongodb-ppc64le:4.0.12\",\"container_image_id\":\"docker.io/ibmcom/icp-mongodb-ppc64le@sha256:f54976512a396366c7947073c656e4bb2b33bd3177ae758a23e3630f37bec7d5\",\"container_iostream\":\"stdout\",\"container_name\":\"acmeair-booking-db\",\"labels\":{\"name\":\"acmeair-booking-db\",\"pod-template-hash\":\"756c74b547\"},\"namespace_id\":\"13f9d1da-d8b3-4c11-8487-929f8ba8bccc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"acme-air\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"acme-air\",\"pod_id\":\"00f03a3b-5d75-4618-a653-b195b5708a1f\",\"pod_ip\":\"10.131.0.213\",\"pod_name\":\"acmeair-booking-db-756c74b547-4vpdt\",\"pod_owner\":\"ReplicaSet/acmeair-booking-db-756c74b547\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"2025-04-16T08:43:08.130+0000 I NETWORK [listener] connection accepted from 10.131.0.218:44428 #3 (3 connections now open)\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-application\"},\"sequence\":1744799089680740215}}"},{"ts":"2025-04-16T10:24:51.700303227Z","line":"{\"@timestamp\":\"2025-04-16T08:42:09.294361444Z\",\"hostname\":\"worker-1\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.213/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:d5\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.213/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.213\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:d5\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://f4541bb456f49222c75388dd8ec10f6e4349ef22f86c3cfb942573912539cb4f\",\"container_image\":\"ibmcom/icp-mongodb-ppc64le:4.0.12\",\"container_image_id\":\"docker.io/ibmcom/icp-mongodb-ppc64le@sha256:f54976512a396366c7947073c656e4bb2b33bd3177ae758a23e3630f37bec7d5\",\"container_iostream\":\"stdout\",\"container_name\":\"acmeair-booking-db\",\"labels\":{\"name\":\"acmeair-booking-db\",\"pod-template-hash\":\"756c74b547\"},\"namespace_id\":\"13f9d1da-d8b3-4c11-8487-929f8ba8bccc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"acme-air\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"acme-air\",\"pod_id\":\"00f03a3b-5d75-4618-a653-b195b5708a1f\",\"pod_ip\":\"10.131.0.213\",\"pod_name\":\"acmeair-booking-db-756c74b547-4vpdt\",\"pod_owner\":\"ReplicaSet/acmeair-booking-db-756c74b547\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"2025-04-16T08:42:09.294+0000 I NETWORK [conn2] received client metadata from 10.131.0.218:57326 conn2: { driver: { name: \\\"mongo-java-driver|legacy\\\", version: \\\"3.10.1\\\" }, os: { type: \\\"Linux\\\", name: \\\"Linux\\\", architecture: \\\"ppc64le\\\", version: \\\"5.14.0-427.62.1.el9_4.ppc64le\\\" }, platform: \\\"Java/Eclipse OpenJ9/1.8.0_262-b10\\\" }\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-application\"},\"sequence\":1744799089680647091}}"},{"ts":"2025-04-16T10:24:51.700254358Z","line":"{\"@timestamp\":\"2025-04-16T08:42:09.293121199Z\",\"hostname\":\"worker-1\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.213/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:d5\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.213/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.213\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:d5\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://f4541bb456f49222c75388dd8ec10f6e4349ef22f86c3cfb942573912539cb4f\",\"container_image\":\"ibmcom/icp-mongodb-ppc64le:4.0.12\",\"container_image_id\":\"docker.io/ibmcom/icp-mongodb-ppc64le@sha256:f54976512a396366c7947073c656e4bb2b33bd3177ae758a23e3630f37bec7d5\",\"container_iostream\":\"stdout\",\"container_name\":\"acmeair-booking-db\",\"labels\":{\"name\":\"acmeair-booking-db\",\"pod-template-hash\":\"756c74b547\"},\"namespace_id\":\"13f9d1da-d8b3-4c11-8487-929f8ba8bccc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"acme-air\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"acme-air\",\"pod_id\":\"00f03a3b-5d75-4618-a653-b195b5708a1f\",\"pod_ip\":\"10.131.0.213\",\"pod_name\":\"acmeair-booking-db-756c74b547-4vpdt\",\"pod_owner\":\"ReplicaSet/acmeair-booking-db-756c74b547\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"2025-04-16T08:42:09.293+0000 I NETWORK [listener] connection accepted from 10.131.0.218:57326 #2 (2 connections now open)\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-application\"},\"sequence\":1744799089680541945}}"},{"ts":"2025-04-16T10:24:51.700216515Z","line":"{\"@timestamp\":\"2025-04-16T08:42:09.209295686Z\",\"hostname\":\"worker-1\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.213/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:d5\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.213/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.213\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:d5\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://f4541bb456f49222c75388dd8ec10f6e4349ef22f86c3cfb942573912539cb4f\",\"container_image\":\"ibmcom/icp-mongodb-ppc64le:4.0.12\",\"container_image_id\":\"docker.io/ibmcom/icp-mongodb-ppc64le@sha256:f54976512a396366c7947073c656e4bb2b33bd3177ae758a23e3630f37bec7d5\",\"container_iostream\":\"stdout\",\"container_name\":\"acmeair-booking-db\",\"labels\":{\"name\":\"acmeair-booking-db\",\"pod-template-hash\":\"756c74b547\"},\"namespace_id\":\"13f9d1da-d8b3-4c11-8487-929f8ba8bccc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"acme-air\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"acme-air\",\"pod_id\":\"00f03a3b-5d75-4618-a653-b195b5708a1f\",\"pod_ip\":\"10.131.0.213\",\"pod_name\":\"acmeair-booking-db-756c74b547-4vpdt\",\"pod_owner\":\"ReplicaSet/acmeair-booking-db-756c74b547\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"2025-04-16T08:42:09.209+0000 I NETWORK [conn1] received client metadata from 10.131.0.218:57324 conn1: { driver: { name: \\\"mongo-java-driver|legacy\\\", version: \\\"3.10.1\\\" }, os: { type: \\\"Linux\\\", name: \\\"Linux\\\", architecture: \\\"ppc64le\\\", version: \\\"5.14.0-427.62.1.el9_4.ppc64le\\\" }, platform: \\\"Java/Eclipse OpenJ9/1.8.0_262-b10\\\" }\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-application\"},\"sequence\":1744799089680462955}}"},{"ts":"2025-04-16T10:24:51.70017445Z","line":"{\"@timestamp\":\"2025-04-16T08:42:09.192335271Z\",\"hostname\":\"worker-1\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.213/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:d5\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.213/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.213\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:d5\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://f4541bb456f49222c75388dd8ec10f6e4349ef22f86c3cfb942573912539cb4f\",\"container_image\":\"ibmcom/icp-mongodb-ppc64le:4.0.12\",\"container_image_id\":\"docker.io/ibmcom/icp-mongodb-ppc64le@sha256:f54976512a396366c7947073c656e4bb2b33bd3177ae758a23e3630f37bec7d5\",\"container_iostream\":\"stdout\",\"container_name\":\"acmeair-booking-db\",\"labels\":{\"name\":\"acmeair-booking-db\",\"pod-template-hash\":\"756c74b547\"},\"namespace_id\":\"13f9d1da-d8b3-4c11-8487-929f8ba8bccc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"acme-air\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"acme-air\",\"pod_id\":\"00f03a3b-5d75-4618-a653-b195b5708a1f\",\"pod_ip\":\"10.131.0.213\",\"pod_name\":\"acmeair-booking-db-756c74b547-4vpdt\",\"pod_owner\":\"ReplicaSet/acmeair-booking-db-756c74b547\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"2025-04-16T08:42:09.192+0000 I NETWORK [listener] connection accepted from 10.131.0.218:57324 #1 (1 connection now open)\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-application\"},\"sequence\":1744799089680353018}}"},{"ts":"2025-04-16T10:24:51.700131658Z","line":"{\"@timestamp\":\"2025-04-16T08:40:54.949955236Z\",\"hostname\":\"worker-1\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.213/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:d5\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.213/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.213\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:d5\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://f4541bb456f49222c75388dd8ec10f6e4349ef22f86c3cfb942573912539cb4f\",\"container_image\":\"ibmcom/icp-mongodb-ppc64le:4.0.12\",\"container_image_id\":\"docker.io/ibmcom/icp-mongodb-ppc64le@sha256:f54976512a396366c7947073c656e4bb2b33bd3177ae758a23e3630f37bec7d5\",\"container_iostream\":\"stdout\",\"container_name\":\"acmeair-booking-db\",\"labels\":{\"name\":\"acmeair-booking-db\",\"pod-template-hash\":\"756c74b547\"},\"namespace_id\":\"13f9d1da-d8b3-4c11-8487-929f8ba8bccc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"acme-air\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"acme-air\",\"pod_id\":\"00f03a3b-5d75-4618-a653-b195b5708a1f\",\"pod_ip\":\"10.131.0.213\",\"pod_name\":\"acmeair-booking-db-756c74b547-4vpdt\",\"pod_owner\":\"ReplicaSet/acmeair-booking-db-756c74b547\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"2025-04-16T08:40:54.949+0000 I INDEX [LogicalSessionCacheRefresh] build index done. scanned 0 total records. 0 secs\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-application\"},\"sequence\":1744799089680271669}}"},{"ts":"2025-04-16T10:24:51.700094752Z","line":"{\"@timestamp\":\"2025-04-16T08:40:54.941706269Z\",\"hostname\":\"worker-1\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.213/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:d5\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.213/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.213\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:d5\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://f4541bb456f49222c75388dd8ec10f6e4349ef22f86c3cfb942573912539cb4f\",\"container_image\":\"ibmcom/icp-mongodb-ppc64le:4.0.12\",\"container_image_id\":\"docker.io/ibmcom/icp-mongodb-ppc64le@sha256:f54976512a396366c7947073c656e4bb2b33bd3177ae758a23e3630f37bec7d5\",\"container_iostream\":\"stdout\",\"container_name\":\"acmeair-booking-db\",\"labels\":{\"name\":\"acmeair-booking-db\",\"pod-template-hash\":\"756c74b547\"},\"namespace_id\":\"13f9d1da-d8b3-4c11-8487-929f8ba8bccc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"acme-air\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"acme-air\",\"pod_id\":\"00f03a3b-5d75-4618-a653-b195b5708a1f\",\"pod_ip\":\"10.131.0.213\",\"pod_name\":\"acmeair-booking-db-756c74b547-4vpdt\",\"pod_owner\":\"ReplicaSet/acmeair-booking-db-756c74b547\"},\"level\":\"default\",\"log_source\":\"container\",\"log_type\":\"application\",\"message\":\"2025-04-16T08:40:54.940+0000 I INDEX [LogicalSessionCacheRefresh] \\t building index using bulk method; build may temporarily use up to 500 megabytes of RAM\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-application\"},\"sequence\":1744799089680193441}}"}]}]} ``` Audit logs: ``` {"ts":"2025-04-16T10:26:21.689402184Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.349+00:00\",\"audit.linux\":{\"record_id\":\"17363\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.349:17363): prog-id=8639 op=UNLOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106924762751}}"},{"ts":"2025-04-16T10:26:21.689348524Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.349+00:00\",\"audit.linux\":{\"record_id\":\"17362\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.349:17362): prog-id=8721 op=LOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106924662431}}"},{"ts":"2025-04-16T10:26:21.689299211Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.349+00:00\",\"audit.linux\":{\"record_id\":\"17361\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.349:17361): prog-id=8638 op=UNLOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106924533084}}"},{"ts":"2025-04-16T10:26:21.689251024Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.349+00:00\",\"audit.linux\":{\"record_id\":\"17360\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.349:17360): prog-id=8637 op=UNLOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106924315221}}"},{"ts":"2025-04-16T10:26:21.689194629Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.349+00:00\",\"audit.linux\":{\"record_id\":\"17359\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.349:17359): prog-id=8720 op=LOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106924211008}}"},{"ts":"2025-04-16T10:26:21.689153667Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.349+00:00\",\"audit.linux\":{\"record_id\":\"17358\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.349:17358): prog-id=8719 op=LOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106924098153}}"},{"ts":"2025-04-16T10:26:21.689106587Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.349+00:00\",\"audit.linux\":{\"record_id\":\"17357\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.349:17357): prog-id=8636 op=UNLOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106923972101}}"},{"ts":"2025-04-16T10:26:21.689052891Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.349+00:00\",\"audit.linux\":{\"record_id\":\"17356\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.349:17356): prog-id=8718 op=LOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106923869624}}"},{"ts":"2025-04-16T10:26:21.68899747Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.349+00:00\",\"audit.linux\":{\"record_id\":\"17355\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.349:17355): prog-id=8635 op=UNLOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106923767263}}"},{"ts":"2025-04-16T10:26:21.68894862Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.349+00:00\",\"audit.linux\":{\"record_id\":\"17354\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.349:17354): prog-id=8717 op=LOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106923662414}}"},{"ts":"2025-04-16T10:26:21.688900044Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.339+00:00\",\"audit.linux\":{\"record_id\":\"17353\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.339:17353): prog-id=8634 op=UNLOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106923561731}}"},{"ts":"2025-04-16T10:26:21.688848624Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.339+00:00\",\"audit.linux\":{\"record_id\":\"17352\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.339:17352): prog-id=8716 op=LOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106923460184}}"},{"ts":"2025-04-16T10:26:21.688801009Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.339+00:00\",\"audit.linux\":{\"record_id\":\"17351\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.339:17351): prog-id=8633 op=UNLOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106923355452}}"},{"ts":"2025-04-16T10:26:21.688745759Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.339+00:00\",\"audit.linux\":{\"record_id\":\"17350\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.339:17350): prog-id=8715 op=LOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106923254087}}"},{"ts":"2025-04-16T10:26:21.688697787Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.339+00:00\",\"audit.linux\":{\"record_id\":\"17349\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.339:17349): prog-id=8632 op=UNLOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106923127056}}"},{"ts":"2025-04-16T10:26:21.688617037Z","line":"{\"@timestamp\":\"2025-04-16T07:17:13.339+00:00\",\"audit.linux\":{\"record_id\":\"17348\",\"type\":\"BPF\"},\"hostname\":\"master-0\",\"kubernetes\":{\"container_name\":\"\",\"namespace_name\":\"\",\"pod_name\":\"\"},\"level\":\"default\",\"log_source\":\"auditd\",\"log_type\":\"audit\",\"message\":\"type=BPF msg=audit(1744787833.339:17348): prog-id=8714 op=LOAD\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-audit\"},\"sequence\":1744799106923014376}}"}]}],"stats":{"summary":{"bytesProcessedPerSecond":420376449,"linesProcessedPerSecond":376028,"totalBytesProcessed":93695562,"totalLinesProcessed":83811,"execTime":0.222884898,"queueTime":0.000089999,"subqueries":0,"totalEntriesReturned":100,"splits":2,"shards":0},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0}}},"ingester":{"totalReached":2,"totalChunksMatched":2,"totalBatches":1,"totalLinesSent":100,"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":301258,"headChunkLines":629,"decompressedBytes":93394304,"decompressedLines":83182,"compressedBytes":7919453,"totalDuplicates":0}}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0}}}} ``` Infrastructure logs: ``` {"ts":"2025-04-16T10:26:21.977019963Z","line":"{\"@timestamp\":\"2025-04-16T09:24:25.417516667Z\",\"hostname\":\"master-0\",\"kubernetes\":{\"annotations\":{\"cluster-autoscaler.kubernetes.io/safe-to-evict\":\"true\",\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.128.1.32/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:80:01:20\\\",\\\"gateway_ips\\\":[\\\"10.128.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"}],\\\"ip_address\\\":\\\"10.128.1.32/23\\\",\\\"gateway_ip\\\":\\\"10.128.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.128.1.32\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:80:01:20\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"operatorframework.io/managed-by\":\"marketplace-operator\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://d4500309554d69d2cf1626631205f2f16c932e93fab036dd1f7684f632bec9c6\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_iostream\":\"stderr\",\"container_name\":\"registry-server\",\"labels\":{\"olm_catalogSource\":\"redhat-marketplace\",\"olm_managed\":\"true\",\"olm_pod-spec-hash\":\"7YYHMlfxPFD69O9Zt6d4EAoOyq34vVlBtaEOj1\"},\"namespace_id\":\"b48cf71f-9abe-4b6d-9a9f-f7f9e7311bdc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-marketplace\",\"olm_operatorgroup_uid_990bb0f0-956b-4206-a933-bee4b2a24816\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"pod-security_kubernetes_io_audit\":\"baseline\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_enforce\":\"baseline\",\"pod-security_kubernetes_io_enforce-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"baseline\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"openshift-marketplace\",\"pod_id\":\"ec67f799-0ef3-48bd-95bf-2ee9a99b392a\",\"pod_ip\":\"10.128.1.32\",\"pod_name\":\"redhat-marketplace-cxfqg\",\"pod_owner\":\"CatalogSource/redhat-marketplace\"},\"level\":\"info\",\"log_source\":\"container\",\"log_type\":\"infrastructure\",\"message\":\"time=\\\"2025-04-16T09:24:25Z\\\" level=info msg=\\\"finished call\\\" :authority=\\\"[redhat-marketplace.openshift-marketplace.svc:50051]\\\" content-type=\\\"[application/grpc]\\\" grpc-accept-encoding=\\\"[gzip]\\\" grpc.code=OK grpc.component=server grpc.method=GetBundleForChannel grpc.method_type=unary grpc.request.deadline=\\\"2025-04-16T09:29:23Z\\\" grpc.service=api.Registry grpc.start_time=\\\"2025-04-16T09:24:25Z\\\" grpc.time_ms=53.063 peer.address=\\\"10.128.1.37:41714\\\" protocol=grpc user-agent=\\\"[grpc-go/1.63.2]\\\"\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-infrastructure\"},\"sequence\":1744799106702092427}}"},{"ts":"2025-04-16T10:26:21.976932117Z","line":"{\"@timestamp\":\"2025-04-16T09:24:25.417460867Z\",\"hostname\":\"master-0\",\"kubernetes\":{\"annotations\":{\"cluster-autoscaler.kubernetes.io/safe-to-evict\":\"true\",\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.128.1.32/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:80:01:20\\\",\\\"gateway_ips\\\":[\\\"10.128.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"}],\\\"ip_address\\\":\\\"10.128.1.32/23\\\",\\\"gateway_ip\\\":\\\"10.128.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.128.1.32\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:80:01:20\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"operatorframework.io/managed-by\":\"marketplace-operator\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://d4500309554d69d2cf1626631205f2f16c932e93fab036dd1f7684f632bec9c6\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_iostream\":\"stderr\",\"container_name\":\"registry-server\",\"labels\":{\"olm_catalogSource\":\"redhat-marketplace\",\"olm_managed\":\"true\",\"olm_pod-spec-hash\":\"7YYHMlfxPFD69O9Zt6d4EAoOyq34vVlBtaEOj1\"},\"namespace_id\":\"b48cf71f-9abe-4b6d-9a9f-f7f9e7311bdc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-marketplace\",\"olm_operatorgroup_uid_990bb0f0-956b-4206-a933-bee4b2a24816\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"pod-security_kubernetes_io_audit\":\"baseline\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_enforce\":\"baseline\",\"pod-security_kubernetes_io_enforce-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"baseline\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"openshift-marketplace\",\"pod_id\":\"ec67f799-0ef3-48bd-95bf-2ee9a99b392a\",\"pod_ip\":\"10.128.1.32\",\"pod_name\":\"redhat-marketplace-cxfqg\",\"pod_owner\":\"CatalogSource/redhat-marketplace\"},\"level\":\"info\",\"log_source\":\"container\",\"log_type\":\"infrastructure\",\"message\":\"time=\\\"2025-04-16T09:24:25Z\\\" level=info msg=\\\"finished call\\\" :authority=\\\"[redhat-marketplace.openshift-marketplace.svc:50051]\\\" content-type=\\\"[application/grpc]\\\" grpc-accept-encoding=\\\"[gzip]\\\" grpc.code=OK grpc.component=server grpc.method=GetBundleForChannel grpc.method_type=unary grpc.request.deadline=\\\"2025-04-16T09:29:23Z\\\" grpc.service=api.Registry grpc.start_time=\\\"2025-04-16T09:24:25Z\\\" grpc.time_ms=21.293 peer.address=\\\"10.128.1.37:41714\\\" protocol=grpc user-agent=\\\"[grpc-go/1.63.2]\\\"\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-infrastructure\"},\"sequence\":1744799106701994262}}"},{"ts":"2025-04-16T10:26:21.976838645Z","line":"{\"@timestamp\":\"2025-04-16T09:24:25.417460867Z\",\"hostname\":\"master-0\",\"kubernetes\":{\"annotations\":{\"cluster-autoscaler.kubernetes.io/safe-to-evict\":\"true\",\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.128.1.32/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:80:01:20\\\",\\\"gateway_ips\\\":[\\\"10.128.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"}],\\\"ip_address\\\":\\\"10.128.1.32/23\\\",\\\"gateway_ip\\\":\\\"10.128.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.128.1.32\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:80:01:20\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"operatorframework.io/managed-by\":\"marketplace-operator\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://d4500309554d69d2cf1626631205f2f16c932e93fab036dd1f7684f632bec9c6\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_iostream\":\"stderr\",\"container_name\":\"registry-server\",\"labels\":{\"olm_catalogSource\":\"redhat-marketplace\",\"olm_managed\":\"true\",\"olm_pod-spec-hash\":\"7YYHMlfxPFD69O9Zt6d4EAoOyq34vVlBtaEOj1\"},\"namespace_id\":\"b48cf71f-9abe-4b6d-9a9f-f7f9e7311bdc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-marketplace\",\"olm_operatorgroup_uid_990bb0f0-956b-4206-a933-bee4b2a24816\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"pod-security_kubernetes_io_audit\":\"baseline\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_enforce\":\"baseline\",\"pod-security_kubernetes_io_enforce-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"baseline\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"openshift-marketplace\",\"pod_id\":\"ec67f799-0ef3-48bd-95bf-2ee9a99b392a\",\"pod_ip\":\"10.128.1.32\",\"pod_name\":\"redhat-marketplace-cxfqg\",\"pod_owner\":\"CatalogSource/redhat-marketplace\"},\"level\":\"info\",\"log_source\":\"container\",\"log_type\":\"infrastructure\",\"message\":\"time=\\\"2025-04-16T09:24:25Z\\\" level=info msg=\\\"started call\\\" :authority=\\\"[redhat-marketplace.openshift-marketplace.svc:50051]\\\" content-type=\\\"[application/grpc]\\\" grpc-accept-encoding=\\\"[gzip]\\\" grpc.component=server grpc.method=GetBundleForChannel grpc.method_type=unary grpc.request.deadline=\\\"2025-04-16T09:29:23Z\\\" grpc.service=api.Registry grpc.start_time=\\\"2025-04-16T09:24:25Z\\\" grpc.time_ms=0.047 peer.address=\\\"10.128.1.37:41714\\\" protocol=grpc user-agent=\\\"[grpc-go/1.63.2]\\\"\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-infrastructure\"},\"sequence\":1744799106701903031}}"},{"ts":"2025-04-16T10:26:21.976747227Z","line":"{\"@timestamp\":\"2025-04-16T09:24:25.417128688Z\",\"hostname\":\"master-0\",\"kubernetes\":{\"annotations\":{\"cluster-autoscaler.kubernetes.io/safe-to-evict\":\"true\",\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.128.1.32/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:80:01:20\\\",\\\"gateway_ips\\\":[\\\"10.128.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"}],\\\"ip_address\\\":\\\"10.128.1.32/23\\\",\\\"gateway_ip\\\":\\\"10.128.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.128.1.32\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:80:01:20\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"operatorframework.io/managed-by\":\"marketplace-operator\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://d4500309554d69d2cf1626631205f2f16c932e93fab036dd1f7684f632bec9c6\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_iostream\":\"stderr\",\"container_name\":\"registry-server\",\"labels\":{\"olm_catalogSource\":\"redhat-marketplace\",\"olm_managed\":\"true\",\"olm_pod-spec-hash\":\"7YYHMlfxPFD69O9Zt6d4EAoOyq34vVlBtaEOj1\"},\"namespace_id\":\"b48cf71f-9abe-4b6d-9a9f-f7f9e7311bdc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-marketplace\",\"olm_operatorgroup_uid_990bb0f0-956b-4206-a933-bee4b2a24816\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"pod-security_kubernetes_io_audit\":\"baseline\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_enforce\":\"baseline\",\"pod-security_kubernetes_io_enforce-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"baseline\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"openshift-marketplace\",\"pod_id\":\"ec67f799-0ef3-48bd-95bf-2ee9a99b392a\",\"pod_ip\":\"10.128.1.32\",\"pod_name\":\"redhat-marketplace-cxfqg\",\"pod_owner\":\"CatalogSource/redhat-marketplace\"},\"level\":\"info\",\"log_source\":\"container\",\"log_type\":\"infrastructure\",\"message\":\"time=\\\"2025-04-16T09:24:25Z\\\" level=info msg=\\\"finished call\\\" :authority=\\\"[redhat-marketplace.openshift-marketplace.svc:50051]\\\" content-type=\\\"[application/grpc]\\\" grpc-accept-encoding=\\\"[gzip]\\\" grpc.code=OK grpc.component=server grpc.method=GetBundleForChannel grpc.method_type=unary grpc.request.deadline=\\\"2025-04-16T09:29:23Z\\\" grpc.service=api.Registry grpc.start_time=\\\"2025-04-16T09:24:25Z\\\" grpc.time_ms=53.882 peer.address=\\\"10.128.1.37:41714\\\" protocol=grpc user-agent=\\\"[grpc-go/1.63.2]\\\"\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-infrastructure\"},\"sequence\":1744799106701818596}}"},{"ts":"2025-04-16T10:26:21.976665829Z","line":"{\"@timestamp\":\"2025-04-16T09:24:25.417001815Z\",\"hostname\":\"master-0\",\"kubernetes\":{\"annotations\":{\"cluster-autoscaler.kubernetes.io/safe-to-evict\":\"true\",\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.128.1.32/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:80:01:20\\\",\\\"gateway_ips\\\":[\\\"10.128.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"}],\\\"ip_address\\\":\\\"10.128.1.32/23\\\",\\\"gateway_ip\\\":\\\"10.128.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.128.1.32\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:80:01:20\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"operatorframework.io/managed-by\":\"marketplace-operator\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://d4500309554d69d2cf1626631205f2f16c932e93fab036dd1f7684f632bec9c6\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_iostream\":\"stderr\",\"container_name\":\"registry-server\",\"labels\":{\"olm_catalogSource\":\"redhat-marketplace\",\"olm_managed\":\"true\",\"olm_pod-spec-hash\":\"7YYHMlfxPFD69O9Zt6d4EAoOyq34vVlBtaEOj1\"},\"namespace_id\":\"b48cf71f-9abe-4b6d-9a9f-f7f9e7311bdc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-marketplace\",\"olm_operatorgroup_uid_990bb0f0-956b-4206-a933-bee4b2a24816\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"pod-security_kubernetes_io_audit\":\"baseline\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_enforce\":\"baseline\",\"pod-security_kubernetes_io_enforce-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"baseline\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"openshift-marketplace\",\"pod_id\":\"ec67f799-0ef3-48bd-95bf-2ee9a99b392a\",\"pod_ip\":\"10.128.1.32\",\"pod_name\":\"redhat-marketplace-cxfqg\",\"pod_owner\":\"CatalogSource/redhat-marketplace\"},\"level\":\"info\",\"log_source\":\"container\",\"log_type\":\"infrastructure\",\"message\":\"time=\\\"2025-04-16T09:24:25Z\\\" level=info msg=\\\"started call\\\" :authority=\\\"[redhat-marketplace.openshift-marketplace.svc:50051]\\\" content-type=\\\"[application/grpc]\\\" grpc-accept-encoding=\\\"[gzip]\\\" grpc.component=server grpc.method=GetBundleForChannel grpc.method_type=unary grpc.request.deadline=\\\"2025-04-16T09:29:23Z\\\" grpc.service=api.Registry grpc.start_time=\\\"2025-04-16T09:24:25Z\\\" grpc.time_ms=5.624 peer.address=\\\"10.128.1.37:41714\\\" protocol=grpc user-agent=\\\"[grpc-go/1.63.2]\\\"\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-infrastructure\"},\"sequence\":1744799106701733870}}"},{"ts":"2025-04-16T10:26:21.976584628Z","line":"{\"@timestamp\":\"2025-04-16T09:24:25.416853284Z\",\"hostname\":\"master-0\",\"kubernetes\":{\"annotations\":{\"cluster-autoscaler.kubernetes.io/safe-to-evict\":\"true\",\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.128.1.32/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:80:01:20\\\",\\\"gateway_ips\\\":[\\\"10.128.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"}],\\\"ip_address\\\":\\\"10.128.1.32/23\\\",\\\"gateway_ip\\\":\\\"10.128.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.128.1.32\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:80:01:20\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"operatorframework.io/managed-by\":\"marketplace-operator\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://d4500309554d69d2cf1626631205f2f16c932e93fab036dd1f7684f632bec9c6\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_iostream\":\"stderr\",\"container_name\":\"registry-server\",\"labels\":{\"olm_catalogSource\":\"redhat-marketplace\",\"olm_managed\":\"true\",\"olm_pod-spec-hash\":\"7YYHMlfxPFD69O9Zt6d4EAoOyq34vVlBtaEOj1\"},\"namespace_id\":\"b48cf71f-9abe-4b6d-9a9f-f7f9e7311bdc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-marketplace\",\"olm_operatorgroup_uid_990bb0f0-956b-4206-a933-bee4b2a24816\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"pod-security_kubernetes_io_audit\":\"baseline\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_enforce\":\"baseline\",\"pod-security_kubernetes_io_enforce-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"baseline\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"openshift-marketplace\",\"pod_id\":\"ec67f799-0ef3-48bd-95bf-2ee9a99b392a\",\"pod_ip\":\"10.128.1.32\",\"pod_name\":\"redhat-marketplace-cxfqg\",\"pod_owner\":\"CatalogSource/redhat-marketplace\"},\"level\":\"info\",\"log_source\":\"container\",\"log_type\":\"infrastructure\",\"message\":\"time=\\\"2025-04-16T09:24:25Z\\\" level=info msg=\\\"started call\\\" :authority=\\\"[redhat-marketplace.openshift-marketplace.svc:50051]\\\" content-type=\\\"[application/grpc]\\\" grpc-accept-encoding=\\\"[gzip]\\\" grpc.component=server grpc.method=GetBundleForChannel grpc.method_type=unary grpc.request.deadline=\\\"2025-04-16T09:29:23Z\\\" grpc.service=api.Registry grpc.start_time=\\\"2025-04-16T09:24:25Z\\\" grpc.time_ms=0.046 peer.address=\\\"10.128.1.37:41714\\\" protocol=grpc user-agent=\\\"[grpc-go/1.63.2]\\\"\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-infrastructure\"},\"sequence\":1744799106701642307}}"},{"ts":"2025-04-16T10:26:21.976502659Z","line":"{\"@timestamp\":\"2025-04-16T09:24:25.416720876Z\",\"hostname\":\"master-0\",\"kubernetes\":{\"annotations\":{\"cluster-autoscaler.kubernetes.io/safe-to-evict\":\"true\",\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.128.1.32/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:80:01:20\\\",\\\"gateway_ips\\\":[\\\"10.128.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"169.254.0.5/32\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"}],\\\"ip_address\\\":\\\"10.128.1.32/23\\\",\\\"gateway_ip\\\":\\\"10.128.0.1\\\",\\\"role\\\":\\\"primary\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.128.1.32\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:80:01:20\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"operatorframework.io/managed-by\":\"marketplace-operator\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://d4500309554d69d2cf1626631205f2f16c932e93fab036dd1f7684f632bec9c6\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb0fadb9a59e86e4b26c1e9c5f646a976fdcf68772b9d533406c2bbddc23dcc4\",\"container_iostream\":\"stderr\",\"container_name\":\"registry-server\",\"labels\":{\"olm_catalogSource\":\"redhat-marketplace\",\"olm_managed\":\"true\",\"olm_pod-spec-hash\":\"7YYHMlfxPFD69O9Zt6d4EAoOyq34vVlBtaEOj1\"},\"namespace_id\":\"b48cf71f-9abe-4b6d-9a9f-f7f9e7311bdc\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-marketplace\",\"olm_operatorgroup_uid_990bb0f0-956b-4206-a933-bee4b2a24816\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"pod-security_kubernetes_io_audit\":\"baseline\",\"pod-security_kubernetes_io_audit-version\":\"latest\",\"pod-security_kubernetes_io_enforce\":\"baseline\",\"pod-security_kubernetes_io_enforce-version\":\"latest\",\"pod-security_kubernetes_io_warn\":\"baseline\",\"pod-security_kubernetes_io_warn-version\":\"latest\"},\"namespace_name\":\"openshift-marketplace\",\"pod_id\":\"ec67f799-0ef3-48bd-95bf-2ee9a99b392a\",\"pod_ip\":\"10.128.1.32\",\"pod_name\":\"redhat-marketplace-cxfqg\",\"pod_owner\":\"CatalogSource/redhat-marketplace\"},\"level\":\"info\",\"log_source\":\"container\",\"log_type\":\"infrastructure\",\"message\":\"time=\\\"2025-04-16T09:24:25Z\\\" level=info msg=\\\"started call\\\" :authority=\\\"[redhat-marketplace.openshift-marketplace.svc:50051]\\\" content-type=\\\"[application/grpc]\\\" grpc-accept-encoding=\\\"[gzip]\\\" grpc.component=server grpc.method=GetBundleForChannel grpc.method_type=unary grpc.request.deadline=\\\"2025-04-16T09:29:23Z\\\" grpc.service=api.Registry grpc.start_time=\\\"2025-04-16T09:24:25Z\\\" grpc.time_ms=0.044 peer.address=\\\"10.128.1.37:41714\\\" protocol=grpc user-agent=\\\"[grpc-go/1.63.2]\\\"\",\"openshift\":{\"cluster_id\":\"d0057771-8074-494e-aee9-f33601c7f37c\",\"labels\":{\"label\":\"6.1_417-infrastructure\"},\"sequence\":1744799106701559389}}"}]}],"stats":{"summary":{"bytesProcessedPerSecond":588457256,"linesProcessedPerSecond":216315,"totalBytesProcessed":34320090,"totalLinesProcessed":12616,"execTime":0.058322146,"queueTime":0.000129999,"subqueries":0,"totalEntriesReturned":100,"splits":2,"shards":0},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0}}},"ingester":{"totalReached":2,"totalChunksMatched":90,"totalBatches":1,"totalLinesSent":100,"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":5903571,"headChunkLines":2704,"decompressedBytes":28416519,"decompressedLines":9912,"compressedBytes":3288956,"totalDuplicates":0}}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0}}}} ```