Cluster logging 5.8.20 on OCP 4.15 with GCP CSV and Pods: ``` [root@rdr-jeev-415log-bastion-0 ~]# oc get csv -n openshift-logging NAME DISPLAY VERSION REPLACES PHASE cluster-logging.v5.8.20 Red Hat OpenShift Logging 5.8.20 cluster-logging.v5.8.19 Succeeded elasticsearch-operator.v5.8.20 OpenShift Elasticsearch Operator 5.8.20 elasticsearch-operator.v5.8.19 Succeeded loki-operator.v5.8.20 Loki Operator 5.8.20 loki-operator.v5.8.19 Succeeded [root@rdr-jeev-415log-bastion-0 ~]# [root@rdr-jeev-415log-bastion-0 ~]# oc get pods -n openshift-logging NAME READY STATUS RESTARTS AGE cluster-logging-operator-59cc8b7874-kvbl2 1/1 Running 0 3h35m collector-4x496 1/1 Running 0 24s collector-4zgt8 1/1 Running 0 24s collector-dt4hf 1/1 Running 0 25s collector-gd8cs 1/1 Running 0 24s collector-zz5j4 1/1 Running 0 24s logging-view-plugin-78dfcf769b-jxwbb 1/1 Running 0 53m lokistack-sample-compactor-0 1/1 Running 0 44m lokistack-sample-distributor-7cd4c4fff8-8vglj 1/1 Running 0 44m lokistack-sample-distributor-7cd4c4fff8-ls48l 1/1 Running 0 44m lokistack-sample-gateway-7cd5bb5b7b-fc9g8 2/2 Running 0 44m lokistack-sample-gateway-7cd5bb5b7b-rmbkb 2/2 Running 0 44m lokistack-sample-index-gateway-0 1/1 Running 0 44m lokistack-sample-index-gateway-1 1/1 Running 0 44m lokistack-sample-ingester-0 1/1 Running 0 44m lokistack-sample-ingester-1 1/1 Running 0 43m lokistack-sample-querier-89d485f44-dw4bp 1/1 Running 0 44m lokistack-sample-querier-89d485f44-tjrlx 1/1 Running 0 44m lokistack-sample-query-frontend-6dccffcb9-49mwd 1/1 Running 0 44m lokistack-sample-query-frontend-6dccffcb9-t5sjk 1/1 Running 0 44m lokistack-sample-ruler-0 1/1 Running 0 44m lokistack-sample-ruler-1 1/1 Running 0 44m [root@rdr-jeev-415log-bastion-0 ~]# ``` Application logs: ``` { "insertId": "476muif68wrmt", "jsonPayload": { "@timestamp": "2025-05-14T12:46:27.540147020Z", "file": "/var/log/pods/test-logging_centos-logtest-bn5bw_deb41613-5199-4569-8d78-7cb401a303f9/centos-logtest/0.log", "hostname": "worker-0", "kubernetes": { "annotations": { "k8s.ovn.org/pod-networks": "{\"default\":{\"ip_addresses\":[\"10.131.0.195/23\"],\"mac_address\":\"0a:58:0a:83:00:c3\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.195/23\",\"gateway_ip\":\"10.131.0.1\"}}", "k8s.v1.cni.cncf.io/network-status": "[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.195\"\n ],\n \"mac\": \"0a:58:0a:83:00:c3\",\n \"default\": true,\n \"dns\": {}\n}]", "openshift.io/scc": "restricted-v2", "seccomp.security.alpha.kubernetes.io/pod": "runtime/default" }, "container_id": "cri-o://d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b", "container_image": "quay.io/pravin_dsilva/centos:logtest", "container_name": "centos-logtest", "labels": { "run": "centos-logtest", "test": "centos-logtest" }, "namespace_id": "e2166d8f-cd66-4fec-b7fe-df5234e1507a", "namespace_labels": { "kubernetes_io_metadata_name": "test-logging", "pod-security_kubernetes_io_audit": "restricted", "pod-security_kubernetes_io_audit-version": "v1.24", "pod-security_kubernetes_io_warn": "restricted", "pod-security_kubernetes_io_warn-version": "v1.24" }, "namespace_name": "test-logging", "pod_id": "deb41613-5199-4569-8d78-7cb401a303f9", "pod_ip": "10.131.0.195", "pod_name": "centos-logtest-bn5bw", "pod_owner": "ReplicationController/centos-logtest" }, "log_type": "application", "message": "{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }", "openshift": { "cluster_id": "ae9a9f3f-409f-486a-9228-008db9788e0c", "sequence": 35 } }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-app", "receiveTimestamp": "2025-05-14T12:46:28.812803683Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "timestamp": "2025-05-14T12:46:28.812803683Z" }, { "insertId": "15e3t4sfdwvsze", "jsonPayload": { "@timestamp": "2025-05-14T12:46:26.538727928Z", "file": "/var/log/pods/test-logging_centos-logtest-bn5bw_deb41613-5199-4569-8d78-7cb401a303f9/centos-logtest/0.log", "hostname": "worker-0", "kubernetes": { "annotations": { "k8s.ovn.org/pod-networks": "{\"default\":{\"ip_addresses\":[\"10.131.0.195/23\"],\"mac_address\":\"0a:58:0a:83:00:c3\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.195/23\",\"gateway_ip\":\"10.131.0.1\"}}", "k8s.v1.cni.cncf.io/network-status": "[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.195\"\n ],\n \"mac\": \"0a:58:0a:83:00:c3\",\n \"default\": true,\n \"dns\": {}\n}]", "openshift.io/scc": "restricted-v2", "seccomp.security.alpha.kubernetes.io/pod": "runtime/default" }, "container_id": "cri-o://d0f1c5e36e95e500e7c4e572c93695cc3931df64d58feb304ea05ea87a89219b", "container_image": "quay.io/pravin_dsilva/centos:logtest", "container_name": "centos-logtest", "labels": { "run": "centos-logtest", "test": "centos-logtest" }, "namespace_id": "e2166d8f-cd66-4fec-b7fe-df5234e1507a", "namespace_labels": { "kubernetes_io_metadata_name": "test-logging", "pod-security_kubernetes_io_audit": "restricted", "pod-security_kubernetes_io_audit-version": "v1.24", "pod-security_kubernetes_io_warn": "restricted", "pod-security_kubernetes_io_warn-version": "v1.24" }, "namespace_name": "test-logging", "pod_id": "deb41613-5199-4569-8d78-7cb401a303f9", "pod_ip": "10.131.0.195", "pod_name": "centos-logtest-bn5bw", "pod_owner": "ReplicationController/centos-logtest" }, "log_type": "application", "message": "{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }", "openshift": { "cluster_id": "ae9a9f3f-409f-486a-9228-008db9788e0c", "sequence": 34 } }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-app", "receiveTimestamp": "2025-05-14T12:46:27.713403811Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "timestamp": "2025-05-14T12:46:27.713403811Z" } ] ``` Audit logs: ``` { "insertId": "1xobq75fh4h7t2", "jsonPayload": { "@timestamp": "2025-05-14T12:48:15.193565764Z", "annotations": { "authorization.k8s.io/decision": "allow", "authorization.k8s.io/reason": "" }, "apiVersion": "audit.k8s.io/v1", "auditID": "0d7ddfc1-a578-48b5-9199-80e2d068ca82", "file": "/var/log/kube-apiserver/audit.log", "hostname": "master-0", "k8s_audit_level": "Metadata", "kind": "Event", "log_type": "audit", "objectRef": { "apiGroup": "coordination.k8s.io", "apiVersion": "v1", "name": "master-1", "namespace": "kube-node-lease", "resource": "leases", "resourceVersion": "5718669", "uid": "ae188fca-f603-434b-be9d-557f123a411f" }, "openshift": { "cluster_id": "ae9a9f3f-409f-486a-9228-008db9788e0c", "sequence": 2265 }, "requestReceivedTimestamp": "2025-05-14T12:48:15.184277Z", "requestURI": "/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/master-1?timeout=10s", "responseStatus": { "code": 200, "metadata": {} }, "sourceIPs": [ "10.20.177.13" ], "source_type": "file", "stage": "ResponseComplete", "stageTimestamp": "2025-05-14T12:48:15.188547Z", "tag": ".k8s-audit.log", "user": { "groups": [ "system:nodes", "system:authenticated" ], "username": "system:node:master-1" }, "userAgent": "kubelet/v1.28.15+f383677 (linux/ppc64le) kubernetes/4cf5291", "verb": "update" }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-audit", "receiveTimestamp": "2025-05-14T12:48:15.948543938Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "master-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "timestamp": "2025-05-14T12:48:15.948543938Z" }, { "insertId": "1xobq75fh4h7t1", "jsonPayload": { "@timestamp": "2025-05-14T12:48:15.059368875Z", "annotations": { "authorization.k8s.io/decision": "allow", "authorization.k8s.io/reason": "" }, "apiVersion": "audit.k8s.io/v1", "auditID": "d2fc7134-5198-46cb-89e6-5bbe844c77c7", "file": "/var/log/kube-apiserver/audit.log", "hostname": "master-0", "k8s_audit_level": "Metadata", "kind": "Event", "log_type": "audit", "objectRef": { "apiGroup": "coordination.k8s.io", "apiVersion": "v1", "name": "worker-0", "namespace": "kube-node-lease", "resource": "leases", "resourceVersion": "5718670", "uid": "551983f2-f9e0-487e-b92e-91b6fe3fc7a5" }, "openshift": { "cluster_id": "ae9a9f3f-409f-486a-9228-008db9788e0c", "sequence": 2264 }, "requestReceivedTimestamp": "2025-05-14T12:48:15.050214Z", "requestURI": "/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/worker-0?timeout=10s", "responseStatus": { "code": 200, "metadata": {} }, "sourceIPs": [ "10.20.177.13" ], "source_type": "file", "stage": "ResponseComplete", "stageTimestamp": "2025-05-14T12:48:15.055144Z", "tag": ".k8s-audit.log", "user": { "groups": [ "system:nodes", "system:authenticated" ], "username": "system:node:worker-0" }, "userAgent": "kubelet/v1.28.15+f383677 (linux/ppc64le) kubernetes/4cf5291", "verb": "update" }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-audit", "receiveTimestamp": "2025-05-14T12:48:15.948543938Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "master-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "timestamp": "2025-05-14T12:48:15.948543938Z" } ] ``` Infrastructure logs: ``` { "insertId": "11qoh2yfikhxut", "jsonPayload": { "@timestamp": "2025-05-14T12:48:46.151047Z", "_RUNTIME_SCOPE": "system", "_STREAM_ID": "0209aada77e34312a762297007f8fa86", "_SYSTEMD_INVOCATION_ID": "92bcf0129f2f4b6cbbe9141517eae89d", "hostname": "worker-0", "log_type": "infrastructure", "message": "time=\"2025-05-14 12:48:46.149497099Z\" level=info msg=\"Image status: &ImageStatusResponse{Image:&Image{Id:e168ff3d7d862ce5f02d84c206b09504fdec8947793e278517be86b258f9b8d0,RepoTags:[],RepoDigests:[quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:546b3635ab9f651032bf142e243809d64391ee7fcdec9c3d0636218c3dd7aa08],Size_:481879806,Uid:nil,Username:,Spec:&ImageSpec{Image:,Annotations:map[string]string{},UserSpecifiedImage:,},Pinned:false,},Info:map[string]string{},}\" id=70787e80-e8a3-4701-b6bf-3fbc1f7861d3 name=/runtime.v1.ImageService/ImageStatus", "openshift": { "cluster_id": "ae9a9f3f-409f-486a-9228-008db9788e0c", "sequence": 570 }, "systemd": { "t": { "BOOT_ID": "d2f6dd306add4edbb730a0e05496eedd", "CAP_EFFECTIVE": "1ffffffffff", "CMDLINE": "/usr/bin/crio", "COMM": "crio", "EXE": "/usr/bin/crio", "GID": "0", "MACHINE_ID": "197daf2d1d694c77b9e3a1dac702c303", "PID": "2889", "SELINUX_CONTEXT": "system_u:system_r:container_runtime_t:s0", "STREAM_ID": "0209aada77e34312a762297007f8fa86", "SYSTEMD_CGROUP": "/system.slice/crio.service", "SYSTEMD_INVOCATION_ID": "92bcf0129f2f4b6cbbe9141517eae89d", "SYSTEMD_SLICE": "system.slice", "SYSTEMD_UNIT": "crio.service", "TRANSPORT": "stdout", "UID": "0" }, "u": { "SYSLOG_FACILITY": "3", "SYSLOG_IDENTIFIER": "crio" } }, "tag": ".journal.system", "time": "2025-05-14T12:48:46+00:00" }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-infra", "receiveTimestamp": "2025-05-14T12:48:47.176625118Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-05-14T12:48:47.176625118Z" }, { "insertId": "11qoh2yfikhxus", "jsonPayload": { "@timestamp": "2025-05-14T12:48:46.149099Z", "_RUNTIME_SCOPE": "system", "_STREAM_ID": "0209aada77e34312a762297007f8fa86", "_SYSTEMD_INVOCATION_ID": "92bcf0129f2f4b6cbbe9141517eae89d", "hostname": "worker-0", "log_type": "infrastructure", "message": "time=\"2025-05-14 12:48:46.148706977Z\" level=info msg=\"Checking image status: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:546b3635ab9f651032bf142e243809d64391ee7fcdec9c3d0636218c3dd7aa08\" id=70787e80-e8a3-4701-b6bf-3fbc1f7861d3 name=/runtime.v1.ImageService/ImageStatus", "openshift": { "cluster_id": "ae9a9f3f-409f-486a-9228-008db9788e0c", "sequence": 569 }, "systemd": { "t": { "BOOT_ID": "d2f6dd306add4edbb730a0e05496eedd", "CAP_EFFECTIVE": "1ffffffffff", "CMDLINE": "/usr/bin/crio", "COMM": "crio", "EXE": "/usr/bin/crio", "GID": "0", "MACHINE_ID": "197daf2d1d694c77b9e3a1dac702c303", "PID": "2889", "SELINUX_CONTEXT": "system_u:system_r:container_runtime_t:s0", "STREAM_ID": "0209aada77e34312a762297007f8fa86", "SYSTEMD_CGROUP": "/system.slice/crio.service", "SYSTEMD_INVOCATION_ID": "92bcf0129f2f4b6cbbe9141517eae89d", "SYSTEMD_SLICE": "system.slice", "SYSTEMD_UNIT": "crio.service", "TRANSPORT": "stdout", "UID": "0" }, "u": { "SYSLOG_FACILITY": "3", "SYSLOG_IDENTIFIER": "crio" } }, "tag": ".journal.system", "time": "2025-05-14T12:48:46+00:00" }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-infra", "receiveTimestamp": "2025-05-14T12:48:47.176625118Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-05-14T12:48:47.176625118Z" } ] ```