Cluster logging 5.8.20 on OCP 4.15 with lokistack CSV and Pods: ``` [root@rdr-jeev-415log-bastion-0 ~]# oc get csv -n openshift-logging NAME DISPLAY VERSION REPLACES PHASE cluster-logging.v5.8.20 Red Hat OpenShift Logging 5.8.20 cluster-logging.v5.8.19 Succeeded elasticsearch-operator.v5.8.20 OpenShift Elasticsearch Operator 5.8.20 elasticsearch-operator.v5.8.19 Succeeded [root@rdr-jeev-415log-bastion-0 ~]# oc get pods -n openshift-logging NAME READY STATUS RESTARTS AGE cluster-logging-operator-59cc8b7874-kvbl2 1/1 Running 0 115m collector-29zk8 1/1 Running 0 106m collector-2qnns 1/1 Running 0 106m collector-fl2tp 1/1 Running 0 106m collector-j9bzb 1/1 Running 0 106m collector-xgnp9 1/1 Running 0 106m elasticsearch-cdm-03gpi55h-1-6dc457c5cb-hg7jp 2/2 Running 0 113m elasticsearch-cdm-03gpi55h-2-85bd7bd5d5-4xn75 2/2 Running 0 113m elasticsearch-cdm-03gpi55h-3-84b869d57d-m2cpq 2/2 Running 0 112m elasticsearch-im-app-29120340-6glgb 0/1 Completed 0 5m35s elasticsearch-im-audit-29120340-g2ccs 0/1 Completed 0 5m35s elasticsearch-im-infra-29120340-5wc7v 0/1 Completed 0 5m35s kibana-65cd7d8f5f-pzchq 2/2 Running 0 112m [root@rdr-jeev-415log-bastion-0 ~]# ``` Lokistack: Infrastructure logs: ``` "{\"@timestamp\":\"2025-05-14T12:03:52.880854501Z\",\"file\":\"/var/log/pods/openshift-machine-api_cluster-autoscaler-operator-7c8bd4bd79-mtkk5_49d02bb2-10af-44f7-8ff3-80944c703c23/cluster-autoscaler-operator/0.log\",\"hostname\":\"master-0\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.128.1.62/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:80:01:3e\\\",\\\"gateway_ips\\\":[\\\"10.128.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"169.254.169.5/32\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.128.0.1\\\"}],\\\"ip_address\\\":\\\"10.128.1.62/23\\\",\\\"gateway_ip\\\":\\\"10.128.0.1\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.128.1.62\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:80:01:3e\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"kubectl.kubernetes.io/default-container\":\"cluster-autoscaler-operator\",\"openshift.io/required-scc\":\"restricted-v2\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://2ae08e65181987629a82f94895655db0c13400a199bf113df3ea03c969f86b41\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cfd6a016e99767656ba97026c5559e174267812da0a776581836d7cdfce1f613\",\"container_name\":\"cluster-autoscaler-operator\",\"labels\":{\"k8s-app\":\"cluster-autoscaler-operator\",\"pod-template-hash\":\"7c8bd4bd79\"},\"namespace_id\":\"c897833b-75c0-4a9e-80ab-4e5c5fef40e9\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-machine-api\",\"name\":\"openshift-machine-api\",\"olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"openshift_io_run-level\":\"\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"namespace_name\":\"openshift-machine-api\",\"pod_id\":\"49d02bb2-10af-44f7-8ff3-80944c703c23\",\"pod_ip\":\"10.128.1.62\",\"pod_name\":\"cluster-autoscaler-operator-7c8bd4bd79-mtkk5\",\"pod_owner\":\"ReplicaSet/cluster-autoscaler-operator-7c8bd4bd79\"},\"level\":\"info\",\"log_type\":\"infrastructure\",\"message\":\"I0514 12:03:52.880816 1 status.go:450] No ClusterAutoscaler. Reporting available.\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":754}}"]]},{"stream":{"k8s_container_name":"approver","k8s_namespace_name":"openshift-network-node-identity","k8s_node_name":"master-2","k8s_pod_name":"network-node-identity-4r45x","kubernetes_container_name":"approver","kubernetes_host":"master-2","kubernetes_namespace_name":"openshift-network-node-identity","kubernetes_pod_name":"network-node-identity-4r45x","log_type":"infrastructure","openshift_log_type":"infrastructure"},"values":[["1747224236800604075","{\"@timestamp\":\"2025-05-14T12:03:52.735426064Z\",\"file\":\"/var/log/pods/openshift-network-node-identity_network-node-identity-4r45x_32c5effc-2841-462b-9691-0a62b3b87e71/approver/0.log\",\"hostname\":\"master-2\",\"kubernetes\":{\"annotations\":{\"openshift.io/scc\":\"hostnetwork-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://f8e23f32f1fa78636d40770b16df245e5d3dfba3aef8411e92f27d2883491988\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718\",\"container_name\":\"approver\",\"labels\":{\"app\":\"network-node-identity\",\"component\":\"network\",\"controller-revision-hash\":\"744b88f845\",\"kubernetes_io_os\":\"linux\",\"openshift_io_component\":\"network\",\"pod-template-generation\":\"1\",\"type\":\"infra\"},\"namespace_id\":\"54063364-133b-4f38-9a16-b16fc653fff9\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-network-node-identity\",\"olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"namespace_name\":\"openshift-network-node-identity\",\"pod_id\":\"32c5effc-2841-462b-9691-0a62b3b87e71\",\"pod_ip\":\"10.20.177.77\",\"pod_name\":\"network-node-identity-4r45x\",\"pod_owner\":\"DaemonSet/network-node-identity\"},\"level\":\"info\",\"log_type\":\"infrastructure\",\"message\":\"I0514 12:03:52.735404 1 leaderelection.go:255] failed to acquire lease openshift-network-node-identity/ovnkube-identity\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":399}}"],["1747224236800515257","{\"@timestamp\":\"2025-05-14T12:03:52.735426064Z\",\"file\":\"/var/log/pods/openshift-network-node-identity_network-node-identity-4r45x_32c5effc-2841-462b-9691-0a62b3b87e71/approver/0.log\",\"hostname\":\"master-2\",\"kubernetes\":{\"annotations\":{\"openshift.io/scc\":\"hostnetwork-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://f8e23f32f1fa78636d40770b16df245e5d3dfba3aef8411e92f27d2883491988\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718\",\"container_name\":\"approver\",\"labels\":{\"app\":\"network-node-identity\",\"component\":\"network\",\"controller-revision-hash\":\"744b88f845\",\"kubernetes_io_os\":\"linux\",\"openshift_io_component\":\"network\",\"pod-template-generation\":\"1\",\"type\":\"infra\"},\"namespace_id\":\"54063364-133b-4f38-9a16-b16fc653fff9\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-network-node-identity\",\"olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"namespace_name\":\"openshift-network-node-identity\",\"pod_id\":\"32c5effc-2841-462b-9691-0a62b3b87e71\",\"pod_ip\":\"10.20.177.77\",\"pod_name\":\"network-node-identity-4r45x\",\"pod_owner\":\"DaemonSet/network-node-identity\"},\"level\":\"info\",\"log_type\":\"infrastructure\",\"message\":\"I0514 12:03:52.735383 1 leaderelection.go:354] lock is held by master-1_cbccd6d3-c481-4d3d-b8a0-45bded6d5f92 and has not yet expired\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":398}}"]]},{"stream":{"k8s_container_name":"approver","k8s_namespace_name":"openshift-network-node-identity","k8s_node_name":"master-0","k8s_pod_name":"network-node-identity-qvgvg","kubernetes_container_name":"approver","kubernetes_host":"master-0","kubernetes_namespace_name":"openshift-network-node-identity","kubernetes_pod_name":"network-node-identity-qvgvg","log_type":"infrastructure","openshift_log_type":"infrastructure"},"values":[["1747224232834354400","{\"@timestamp\":\"2025-05-14T12:03:45.500248574Z\",\"file\":\"/var/log/pods/openshift-network-node-identity_network-node-identity-qvgvg_095a508a-03aa-4c3d-b794-27c6a40bc1f3/approver/0.log\",\"hostname\":\"master-0\",\"kubernetes\":{\"annotations\":{\"openshift.io/scc\":\"hostnetwork-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://226437ce4cd12b9c1c58ba2ea773a6a6c0f88673f405f7e76ef938328b58aeb9\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718\",\"container_name\":\"approver\",\"labels\":{\"app\":\"network-node-identity\",\"component\":\"network\",\"controller-revision-hash\":\"744b88f845\",\"kubernetes_io_os\":\"linux\",\"openshift_io_component\":\"network\",\"pod-template-generation\":\"1\",\"type\":\"infra\"},\"namespace_id\":\"54063364-133b-4f38-9a16-b16fc653fff9\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-network-node-identity\",\"olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"namespace_name\":\"openshift-network-node-identity\",\"pod_id\":\"095a508a-03aa-4c3d-b794-27c6a40bc1f3\",\"pod_ip\":\"10.20.177.64\",\"pod_name\":\"network-node-identity-qvgvg\",\"pod_owner\":\"DaemonSet/network-node-identity\"},\"level\":\"info\",\"log_type\":\"infrastructure\",\"message\":\"I0514 12:03:45.500227 1 leaderelection.go:255] failed to acquire lease openshift-network-node-identity/ovnkube-identity\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":730}}"],["1747224232834281449","{\"@timestamp\":\"2025-05-14T12:03:45.500248574Z\",\"file\":\"/var/log/pods/openshift-network-node-identity_network-node-identity-qvgvg_095a508a-03aa-4c3d-b794-27c6a40bc1f3/approver/0.log\",\"hostname\":\"master-0\",\"kubernetes\":{\"annotations\":{\"openshift.io/scc\":\"hostnetwork-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://226437ce4cd12b9c1c58ba2ea773a6a6c0f88673f405f7e76ef938328b58aeb9\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7d779b47a4bd00951533a66ed4ae86d14892fa187f6cf604ef2a5f08af1b2718\",\"container_name\":\"approver\",\"labels\":{\"app\":\"network-node-identity\",\"component\":\"network\",\"controller-revision-hash\":\"744b88f845\",\"kubernetes_io_os\":\"linux\",\"openshift_io_component\":\"network\",\"pod-template-generation\":\"1\",\"type\":\"infra\"},\"namespace_id\":\"54063364-133b-4f38-9a16-b16fc653fff9\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-network-node-identity\",\"olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"namespace_name\":\"openshift-network-node-identity\",\"pod_id\":\"095a508a-03aa-4c3d-b794-27c6a40bc1f3\",\"pod_ip\":\"10.20.177.64\",\"pod_name\":\"network-node-identity-qvgvg\",\"pod_owner\":\"DaemonSet/network-node-identity\"},\"level\":\"info\",\"log_type\":\"infrastructure\",\"message\":\"I0514 12:03:45.500202 1 leaderelection.go:354] lock is held by master-1_cbccd6d3-c481-4d3d-b8a0-45bded6d5f92 and has not yet expired\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":729}}"]]}],"stats":{"summary":{"bytesProcessedPerSecond":1499268,"linesProcessedPerSecond":734,"totalBytesProcessed":4239499,"totalLinesProcessed":2076,"execTime":2.827712,"queueTime":0,"subqueries":0,"totalEntriesReturned":100,"splits":1,"shards":16,"totalPostFilterLines":2076,"totalStructuredMetadataBytesProcessed":0},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"queryReferencedStructuredMetadata":false,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0,"postFilterLines":0,"headChunkStructuredMetadataBytes":0,"decompressedStructuredMetadataBytes":0},"chunkRefsFetchTime":11021081634,"congestionControlLatency":0,"pipelineWrapperFilteredLines":0}},"ingester":{"totalReached":32,"totalChunksMatched":74,"totalBatches":20,"totalLinesSent":1251,"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"queryReferencedStructuredMetadata":false,"chunk":{"headChunkBytes":3435285,"headChunkLines":1726,"decompressedBytes":804214,"decompressedLines":350,"compressedBytes":335373,"totalDuplicates":0,"postFilterLines":2076,"headChunkStructuredMetadataBytes":0,"decompressedStructuredMetadataBytes":0},"chunkRefsFetchTime":0,"congestionControlLatency":0,"pipelineWrapperFilteredLines":0}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"statsResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"volumeResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"seriesResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"labelResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"instantMetricResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0}},"index":{"totalChunks":0,"postFilterChunks":0,"shardsDuration":0,"usedBloomFilters":false}}}} ``` Audit logs: ``` "{\"@timestamp\":\"2025-05-14T12:05:47.818371977Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"cluster-network-operator\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"cluster-network-operator/openshift-network-operator\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"87096ae7-335b-4e58-ad76-bb428a45ceee\",\"file\":\"/var/log/kube-apiserver/audit.log\",\"hostname\":\"master-0\",\"k8s_audit_level\":\"Metadata\",\"kind\":\"Event\",\"level\":\"Metadata\",\"log_type\":\"audit\",\"objectRef\":{\"apiGroup\":\"controlplane.operator.openshift.io\",\"apiVersion\":\"v1alpha1\",\"name\":\"network-check-source-worker-0-to-kubernetes-apiserver-endpoint-master-0\",\"namespace\":\"openshift-network-diagnostics\",\"resource\":\"podnetworkconnectivitychecks\"},\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":158847},\"requestReceivedTimestamp\":\"2025-05-14T12:05:47.735921Z\",\"requestURI\":\"/apis/controlplane.operator.openshift.io/v1alpha1/namespaces/openshift-network-diagnostics/podnetworkconnectivitychecks/network-check-source-worker-0-to-kubernetes-apiserver-endpoint-master-0?fieldManager=ConnectivityCheckController\u0026force=true\",\"responseStatus\":{\"code\":200,\"metadata\":{}},\"sourceIPs\":[\"10.20.177.13\"],\"source_type\":\"file\",\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-14T12:05:47.807310Z\",\"user\":{\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"network-operator-768fcf7769-xzl99\"],\"authentication.kubernetes.io/pod-uid\":[\"c268c47e-c067-44a5-b366-34cd85e7e54f\"]},\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-network-operator\",\"system:authenticated\"],\"uid\":\"2ee45e73-b827-4896-a9ae-e675787b8e3c\",\"username\":\"system:serviceaccount:openshift-network-operator:cluster-network-operator\"},\"userAgent\":\"cluster-network-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format\",\"verb\":\"patch\"}"],["1747224347820520297","{\"@timestamp\":\"2025-05-14T12:05:47.818344491Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:openshift:oauth-apiserver\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"oauth-apiserver-sa/openshift-oauth-apiserver\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"8fec36a6-33bb-4228-9128-c5e6cf7fd12a\",\"file\":\"/var/log/kube-apiserver/audit.log\",\"hostname\":\"master-0\",\"k8s_audit_level\":\"Metadata\",\"kind\":\"Event\",\"level\":\"Metadata\",\"log_type\":\"audit\",\"objectRef\":{\"apiVersion\":\"v1\",\"name\":\"kubeadmin\",\"namespace\":\"kube-system\",\"resource\":\"secrets\"},\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":158846},\"requestReceivedTimestamp\":\"2025-05-14T12:05:47.769078Z\",\"requestURI\":\"/api/v1/namespaces/kube-system/secrets/kubeadmin\",\"responseStatus\":{\"code\":200,\"metadata\":{}},\"sourceIPs\":[\"10.20.177.159\"],\"source_type\":\"file\",\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-14T12:05:47.805631Z\",\"user\":{\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"apiserver-5548d847fb-6mrwk\"],\"authentication.kubernetes.io/pod-uid\":[\"1a0ca5e4-34bc-4e87-8159-5ba5e5243ebb\"]},\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-oauth-apiserver\",\"system:authenticated\"],\"uid\":\"bbd42af1-3995-431a-bd71-a3c774166732\",\"username\":\"system:serviceaccount:openshift-oauth-apiserver:oauth-apiserver-sa\"},\"userAgent\":\"oauth-apiserver/v0.0.0 (linux/ppc64le) kubernetes/$Format\",\"verb\":\"get\"}"],["1747224347820467233","{\"@timestamp\":\"2025-05-14T12:05:47.818314845Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:openshift:openshift-apiserver\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"openshift-apiserver-sa/openshift-apiserver\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"a5c41690-85f1-41c0-a9f8-25f3a5bbb281\",\"file\":\"/var/log/kube-apiserver/audit.log\",\"hostname\":\"master-0\",\"k8s_audit_level\":\"Metadata\",\"kind\":\"Event\",\"level\":\"Metadata\",\"log_type\":\"audit\",\"objectRef\":{\"apiGroup\":\"authorization.k8s.io\",\"apiVersion\":\"v1\",\"resource\":\"subjectaccessreviews\"},\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":158845},\"requestReceivedTimestamp\":\"2025-05-14T12:05:47.786338Z\",\"requestURI\":\"/apis/authorization.k8s.io/v1/subjectaccessreviews?timeout=10s\",\"responseStatus\":{\"code\":201,\"metadata\":{}},\"sourceIPs\":[\"10.20.177.77\"],\"source_type\":\"file\",\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-14T12:05:47.788007Z\",\"user\":{\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"apiserver-6bd8bfc7df-pvzz6\"],\"authentication.kubernetes.io/pod-uid\":[\"8db37b33-7067-4661-87d4-b2193acdbb5d\"]},\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-apiserver\",\"system:authenticated\"],\"uid\":\"ec49b816-ebc9-4fed-8745-be7929678376\",\"username\":\"system:serviceaccount:openshift-apiserver:openshift-apiserver-sa\"},\"userAgent\":\"openshift-apiserver/v0.0.0 (linux/ppc64le) kubernetes/$Format\",\"verb\":\"create\"}"],["1747224347820412701","{\"@timestamp\":\"2025-05-14T12:05:47.818262833Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:openshift:openshift-apiserver\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"openshift-apiserver-sa/openshift-apiserver\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"f99949c2-caea-45a2-8dfc-429208df194b\",\"file\":\"/var/log/kube-apiserver/audit.log\",\"hostname\":\"master-0\",\"k8s_audit_level\":\"Metadata\",\"kind\":\"Event\",\"level\":\"Metadata\",\"log_type\":\"audit\",\"objectRef\":{\"apiGroup\":\"authorization.k8s.io\",\"apiVersion\":\"v1\",\"resource\":\"subjectaccessreviews\"},\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":158844},\"requestReceivedTimestamp\":\"2025-05-14T12:05:47.786116Z\",\"requestURI\":\"/apis/authorization.k8s.io/v1/subjectaccessreviews?timeout=10s\",\"responseStatus\":{\"code\":201,\"metadata\":{}},\"sourceIPs\":[\"10.20.177.77\"],\"source_type\":\"file\",\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-14T12:05:47.787792Z\",\"user\":{\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"apiserver-6bd8bfc7df-pvzz6\"],\"authentication.kubernetes.io/pod-uid\":[\"8db37b33-7067-4661-87d4-b2193acdbb5d\"]},\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-apiserver\",\"system:authenticated\"],\"uid\":\"ec49b816-ebc9-4fed-8745-be7929678376\",\"username\":\"system:serviceaccount:openshift-apiserver:openshift-apiserver-sa\"},\"userAgent\":\"openshift-apiserver/v0.0.0 (linux/ppc64le) kubernetes/$Format\",\"verb\":\"create\"}"],["1747224347820321018","{\"@timestamp\":\"2025-05-14T12:05:47.818199929Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:openshift:openshift-apiserver\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"openshift-apiserver-sa/openshift-apiserver\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"2d795ddc-31ed-4e48-90a2-a2652c947988\",\"file\":\"/var/log/kube-apiserver/audit.log\",\"hostname\":\"master-0\",\"k8s_audit_level\":\"Metadata\",\"kind\":\"Event\",\"level\":\"Metadata\",\"log_type\":\"audit\",\"objectRef\":{\"apiGroup\":\"authorization.k8s.io\",\"apiVersion\":\"v1\",\"resource\":\"subjectaccessreviews\"},\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":158843},\"requestReceivedTimestamp\":\"2025-05-14T12:05:47.784909Z\",\"requestURI\":\"/apis/authorization.k8s.io/v1/subjectaccessreviews?timeout=10s\",\"responseStatus\":{\"code\":201,\"metadata\":{}},\"sourceIPs\":[\"10.20.177.77\"],\"source_type\":\"file\",\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-14T12:05:47.786777Z\",\"user\":{\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"apiserver-6bd8bfc7df-pvzz6\"],\"authentication.kubernetes.io/pod-uid\":[\"8db37b33-7067-4661-87d4-b2193acdbb5d\"]},\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-apiserver\",\"system:authenticated\"],\"uid\":\"ec49b816-ebc9-4fed-8745-be7929678376\",\"username\":\"system:serviceaccount:openshift-apiserver:openshift-apiserver-sa\"},\"userAgent\":\"openshift-apiserver/v0.0.0 (linux/ppc64le) kubernetes/$Format\",\"verb\":\"create\"}"],["1747224347751514704","{\"@timestamp\":\"2025-05-14T12:05:47.751083716Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:openshift:openshift-apiserver\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"openshift-apiserver-sa/openshift-apiserver\\\"\"},\"apiVersion\":\"audit.k8s.io/v1\",\"auditID\":\"22adb00e-513c-4a39-83c6-b6fd7e19f208\",\"file\":\"/var/log/kube-apiserver/audit.log\",\"hostname\":\"master-0\",\"k8s_audit_level\":\"Metadata\",\"kind\":\"Event\",\"level\":\"Metadata\",\"log_type\":\"audit\",\"objectRef\":{\"apiGroup\":\"authorization.k8s.io\",\"apiVersion\":\"v1\",\"resource\":\"subjectaccessreviews\"},\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":158842},\"requestReceivedTimestamp\":\"2025-05-14T12:05:47.742882Z\",\"requestURI\":\"/apis/authorization.k8s.io/v1/subjectaccessreviews?timeout=10s\",\"responseStatus\":{\"code\":201,\"metadata\":{}},\"sourceIPs\":[\"10.20.177.77\"],\"source_type\":\"file\",\"stage\":\"ResponseComplete\",\"stageTimestamp\":\"2025-05-14T12:05:47.744903Z\",\"user\":{\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"apiserver-6bd8bfc7df-pvzz6\"],\"authentication.kubernetes.io/pod-uid\":[\"8db37b33-7067-4661-87d4-b2193acdbb5d\"]},\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-apiserver\",\"system:authenticated\"],\"uid\":\"ec49b816-ebc9-4fed-8745-be7929678376\",\"username\":\"system:serviceaccount:openshift-apiserver:openshift-apiserver-sa\"},\"userAgent\":\"openshift-apiserver/v0.0.0 (linux/ppc64le) kubernetes/$Format\",\"verb\":\"create\"}"]]}],"stats":{"summary":{"bytesProcessedPerSecond":19963571,"linesProcessedPerSecond":12941,"totalBytesProcessed":112441715,"totalLinesProcessed":72893,"execTime":5.632345,"queueTime":0,"subqueries":0,"totalEntriesReturned":100,"splits":1,"shards":16,"totalPostFilterLines":72893,"totalStructuredMetadataBytesProcessed":0},"querier":{"store":{"totalChunksRef":52,"totalChunksDownloaded":52,"chunksDownloadTime":17039405291,"queryReferencedStructuredMetadata":false,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":5263280,"decompressedLines":3277,"compressedBytes":98178235,"totalDuplicates":0,"postFilterLines":3277,"headChunkStructuredMetadataBytes":0,"decompressedStructuredMetadataBytes":0},"chunkRefsFetchTime":3700354276,"congestionControlLatency":0,"pipelineWrapperFilteredLines":0}},"ingester":{"totalReached":32,"totalChunksMatched":85,"totalBatches":16,"totalLinesSent":1400,"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"queryReferencedStructuredMetadata":false,"chunk":{"headChunkBytes":3531846,"headChunkLines":2696,"decompressedBytes":103646589,"decompressedLines":66920,"compressedBytes":121625861,"totalDuplicates":0,"postFilterLines":69616,"headChunkStructuredMetadataBytes":0,"decompressedStructuredMetadataBytes":0},"chunkRefsFetchTime":0,"congestionControlLatency":0,"pipelineWrapperFilteredLines":0}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":52,"entriesStored":104,"bytesReceived":0,"bytesSent":98249392,"requests":30,"downloadTime":283354,"queryLengthServed":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"statsResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"volumeResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"seriesResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"labelResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"instantMetricResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0}},"index":{"totalChunks":0,"postFilterChunks":0,"shardsDuration":0,"usedBloomFilters":false}}}} ``` Application logs: ``` {\"@timestamp\":\"2025-05-14T12:14:51.626309239Z\",\"file\":\"/var/log/pods/test-logging_centos-logtest-bn5bw_deb41613-5199-4569-8d78-7cb401a303f9/centos-logtest/0.log\",\"hostname\":\"worker-0\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.195/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:c3\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.169.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.195/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.195\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:c3\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_name\":\"centos-logtest\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"namespace_id\":\"e2166d8f-cd66-4fec-b7fe-df5234e1507a\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"test-logging\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"namespace_name\":\"test-logging\",\"pod_id\":\"deb41613-5199-4569-8d78-7cb401a303f9\",\"pod_ip\":\"10.131.0.195\",\"pod_name\":\"centos-logtest-bn5bw\",\"pod_owner\":\"ReplicationController/centos-logtest\"},\"level\":\"default\",\"log_type\":\"application\",\"message\":\"{\\\"message\\\": \\\"MERGE_JSON_LOG=true\\\", \\\"level\\\": \\\"debug\\\",\\\"Layer1\\\": \\\"layer1 0\\\", \\\"layer2\\\": {\\\"name\\\":\\\"Layer2 1\\\", \\\"tips\\\":\\\"Decide by PRESERVE_JSON_LOG\\\"}, \\\"StringNumber\\\":\\\"10\\\", \\\"Number\\\": 10,\\\"foo.bar\\\":\\\"Dot Item\\\",\\\"{foobar}\\\":\\\"Brace Item\\\",\\\"[foobar]\\\":\\\"Bracket Item\\\", \\\"foo:bar\\\":\\\"Colon Item\\\",\\\"foo bar\\\":\\\"Space Item\\\" }\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":785}}"],["1747224890800740061","{\"@timestamp\":\"2025-05-14T12:14:50.624806084Z\",\"file\":\"/var/log/pods/test-logging_centos-logtest-bn5bw_deb41613-5199-4569-8d78-7cb401a303f9/centos-logtest/0.log\",\"hostname\":\"worker-0\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.195/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:c3\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.169.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.195/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.195\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:c3\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_name\":\"centos-logtest\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"namespace_id\":\"e2166d8f-cd66-4fec-b7fe-df5234e1507a\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"test-logging\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"namespace_name\":\"test-logging\",\"pod_id\":\"deb41613-5199-4569-8d78-7cb401a303f9\",\"pod_ip\":\"10.131.0.195\",\"pod_name\":\"centos-logtest-bn5bw\",\"pod_owner\":\"ReplicationController/centos-logtest\"},\"level\":\"default\",\"log_type\":\"application\",\"message\":\"{\\\"message\\\": \\\"MERGE_JSON_LOG=true\\\", \\\"level\\\": \\\"debug\\\",\\\"Layer1\\\": \\\"layer1 0\\\", \\\"layer2\\\": {\\\"name\\\":\\\"Layer2 1\\\", \\\"tips\\\":\\\"Decide by PRESERVE_JSON_LOG\\\"}, \\\"StringNumber\\\":\\\"10\\\", \\\"Number\\\": 10,\\\"foo.bar\\\":\\\"Dot Item\\\",\\\"{foobar}\\\":\\\"Brace Item\\\",\\\"[foobar]\\\":\\\"Bracket Item\\\", \\\"foo:bar\\\":\\\"Colon Item\\\",\\\"foo bar\\\":\\\"Space Item\\\" }\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":784}}"],["1747224889765918632","{\"@timestamp\":\"2025-05-14T12:14:49.623252075Z\",\"file\":\"/var/log/pods/test-logging_centos-logtest-bn5bw_deb41613-5199-4569-8d78-7cb401a303f9/centos-logtest/0.log\",\"hostname\":\"worker-0\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.195/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:c3\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.169.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.195/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.195\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:c3\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_name\":\"centos-logtest\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"namespace_id\":\"e2166d8f-cd66-4fec-b7fe-df5234e1507a\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"test-logging\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"namespace_name\":\"test-logging\",\"pod_id\":\"deb41613-5199-4569-8d78-7cb401a303f9\",\"pod_ip\":\"10.131.0.195\",\"pod_name\":\"centos-logtest-bn5bw\",\"pod_owner\":\"ReplicationController/centos-logtest\"},\"level\":\"default\",\"log_type\":\"application\",\"message\":\"{\\\"message\\\": \\\"MERGE_JSON_LOG=true\\\", \\\"level\\\": \\\"debug\\\",\\\"Layer1\\\": \\\"layer1 0\\\", \\\"layer2\\\": {\\\"name\\\":\\\"Layer2 1\\\", \\\"tips\\\":\\\"Decide by PRESERVE_JSON_LOG\\\"}, \\\"StringNumber\\\":\\\"10\\\", \\\"Number\\\": 10,\\\"foo.bar\\\":\\\"Dot Item\\\",\\\"{foobar}\\\":\\\"Brace Item\\\",\\\"[foobar]\\\":\\\"Bracket Item\\\", \\\"foo:bar\\\":\\\"Colon Item\\\",\\\"foo bar\\\":\\\"Space Item\\\" }\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":783}}"],["1747224888731338702","{\"@timestamp\":\"2025-05-14T12:14:48.622261719Z\",\"file\":\"/var/log/pods/test-logging_centos-logtest-bn5bw_deb41613-5199-4569-8d78-7cb401a303f9/centos-logtest/0.log\",\"hostname\":\"worker-0\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.195/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:c3\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.169.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.195/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.195\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:c3\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_name\":\"centos-logtest\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"namespace_id\":\"e2166d8f-cd66-4fec-b7fe-df5234e1507a\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"test-logging\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"namespace_name\":\"test-logging\",\"pod_id\":\"deb41613-5199-4569-8d78-7cb401a303f9\",\"pod_ip\":\"10.131.0.195\",\"pod_name\":\"centos-logtest-bn5bw\",\"pod_owner\":\"ReplicationController/centos-logtest\"},\"level\":\"default\",\"log_type\":\"application\",\"message\":\"{\\\"message\\\": \\\"MERGE_JSON_LOG=true\\\", \\\"level\\\": \\\"debug\\\",\\\"Layer1\\\": \\\"layer1 0\\\", \\\"layer2\\\": {\\\"name\\\":\\\"Layer2 1\\\", \\\"tips\\\":\\\"Decide by PRESERVE_JSON_LOG\\\"}, \\\"StringNumber\\\":\\\"10\\\", \\\"Number\\\": 10,\\\"foo.bar\\\":\\\"Dot Item\\\",\\\"{foobar}\\\":\\\"Brace Item\\\",\\\"[foobar]\\\":\\\"Bracket Item\\\", \\\"foo:bar\\\":\\\"Colon Item\\\",\\\"foo bar\\\":\\\"Space Item\\\" }\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":782}}"],["1747224887624367052","{\"@timestamp\":\"2025-05-14T12:14:47.620768585Z\",\"file\":\"/var/log/pods/test-logging_centos-logtest-bn5bw_deb41613-5199-4569-8d78-7cb401a303f9/centos-logtest/0.log\",\"hostname\":\"worker-0\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.195/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:c3\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.169.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.195/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.195\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:c3\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_name\":\"centos-logtest\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"namespace_id\":\"e2166d8f-cd66-4fec-b7fe-df5234e1507a\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"test-logging\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"namespace_name\":\"test-logging\",\"pod_id\":\"deb41613-5199-4569-8d78-7cb401a303f9\",\"pod_ip\":\"10.131.0.195\",\"pod_name\":\"centos-logtest-bn5bw\",\"pod_owner\":\"ReplicationController/centos-logtest\"},\"level\":\"default\",\"log_type\":\"application\",\"message\":\"{\\\"message\\\": \\\"MERGE_JSON_LOG=true\\\", \\\"level\\\": \\\"debug\\\",\\\"Layer1\\\": \\\"layer1 0\\\", \\\"layer2\\\": {\\\"name\\\":\\\"Layer2 1\\\", \\\"tips\\\":\\\"Decide by PRESERVE_JSON_LOG\\\"}, \\\"StringNumber\\\":\\\"10\\\", \\\"Number\\\": 10,\\\"foo.bar\\\":\\\"Dot Item\\\",\\\"{foobar}\\\":\\\"Brace Item\\\",\\\"[foobar]\\\":\\\"Bracket Item\\\", \\\"foo:bar\\\":\\\"Colon Item\\\",\\\"foo bar\\\":\\\"Space Item\\\" }\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":781}}"],["1747224887102658812","{\"@timestamp\":\"2025-05-14T12:14:46.619378059Z\",\"file\":\"/var/log/pods/test-logging_centos-logtest-bn5bw_deb41613-5199-4569-8d78-7cb401a303f9/centos-logtest/0.log\",\"hostname\":\"worker-0\",\"kubernetes\":{\"annotations\":{\"k8s.ovn.org/pod-networks\":\"{\\\"default\\\":{\\\"ip_addresses\\\":[\\\"10.131.0.195/23\\\"],\\\"mac_address\\\":\\\"0a:58:0a:83:00:c3\\\",\\\"gateway_ips\\\":[\\\"10.131.0.1\\\"],\\\"routes\\\":[{\\\"dest\\\":\\\"10.128.0.0/14\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"172.30.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"169.254.169.5/32\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"},{\\\"dest\\\":\\\"100.64.0.0/16\\\",\\\"nextHop\\\":\\\"10.131.0.1\\\"}],\\\"ip_address\\\":\\\"10.131.0.195/23\\\",\\\"gateway_ip\\\":\\\"10.131.0.1\\\"}}\",\"k8s.v1.cni.cncf.io/network-status\":\"[{\\n \\\"name\\\": \\\"ovn-kubernetes\\\",\\n \\\"interface\\\": \\\"eth0\\\",\\n \\\"ips\\\": [\\n \\\"10.131.0.195\\\"\\n ],\\n \\\"mac\\\": \\\"0a:58:0a:83:00:c3\\\",\\n \\\"default\\\": true,\\n \\\"dns\\\": {}\\n}]\",\"openshift.io/scc\":\"restricted-v2\",\"seccomp.security.alpha.kubernetes.io/pod\":\"runtime/default\"},\"container_id\":\"cri-o://d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_name\":\"centos-logtest\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"namespace_id\":\"e2166d8f-cd66-4fec-b7fe-df5234e1507a\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"test-logging\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"namespace_name\":\"test-logging\",\"pod_id\":\"deb41613-5199-4569-8d78-7cb401a303f9\",\"pod_ip\":\"10.131.0.195\",\"pod_name\":\"centos-logtest-bn5bw\",\"pod_owner\":\"ReplicationController/centos-logtest\"},\"level\":\"default\",\"log_type\":\"application\",\"message\":\"{\\\"message\\\": \\\"MERGE_JSON_LOG=true\\\", \\\"level\\\": \\\"debug\\\",\\\"Layer1\\\": \\\"layer1 0\\\", \\\"layer2\\\": {\\\"name\\\":\\\"Layer2 1\\\", \\\"tips\\\":\\\"Decide by PRESERVE_JSON_LOG\\\"}, \\\"StringNumber\\\":\\\"10\\\", \\\"Number\\\": 10,\\\"foo.bar\\\":\\\"Dot Item\\\",\\\"{foobar}\\\":\\\"Brace Item\\\",\\\"[foobar]\\\":\\\"Bracket Item\\\", \\\"foo:bar\\\":\\\"Colon Item\\\",\\\"foo bar\\\":\\\"Space Item\\\" }\",\"openshift\":{\"cluster_id\":\"ae9a9f3f-409f-486a-9228-008db9788e0c\",\"sequence\":780}}"]]}],"stats":{"summary":{"bytesProcessedPerSecond":295551,"linesProcessedPerSecond":131,"totalBytesProcessed":383476,"totalLinesProcessed":171,"execTime":1.297495,"queueTime":0,"subqueries":0,"totalEntriesReturned":100,"splits":1,"shards":16,"totalPostFilterLines":171,"totalStructuredMetadataBytesProcessed":0},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"queryReferencedStructuredMetadata":false,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0,"postFilterLines":0,"headChunkStructuredMetadataBytes":0,"decompressedStructuredMetadataBytes":0},"chunkRefsFetchTime":5060558318,"congestionControlLatency":0,"pipelineWrapperFilteredLines":0}},"ingester":{"totalReached":32,"totalChunksMatched":2,"totalBatches":17,"totalLinesSent":114,"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"queryReferencedStructuredMetadata":false,"chunk":{"headChunkBytes":117504,"headChunkLines":53,"decompressedBytes":265972,"decompressedLines":118,"compressedBytes":150762,"totalDuplicates":0,"postFilterLines":171,"headChunkStructuredMetadataBytes":0,"decompressedStructuredMetadataBytes":0},"chunkRefsFetchTime":0,"congestionControlLatency":0,"pipelineWrapperFilteredLines":0}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"statsResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"volumeResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"seriesResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"labelResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"instantMetricResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0}},"index":{"totalChunks":0,"postFilterChunks":0,"shardsDuration":0,"usedBloomFilters":false}}}} ```