Cluster logging 6.0.8 on OCP 4.15 with GCP CSV and Pods: ``` [root@rdr-jeev-415log-bastion-0 ~]# oc get csv -n openshift-logging NAME DISPLAY VERSION REPLACES PHASE cluster-logging.v6.0.8 Red Hat OpenShift Logging 6.0.8 cluster-logging.v6.0.7 Succeeded loki-operator.v6.0.8 Loki Operator 6.0.8 loki-operator.v6.0.7 Succeeded [root@rdr-jeev-415log-bastion-0 ~]# [root@rdr-jeev-415log-bastion-0 ~]# oc get pods -n openshift-logging NAME READY STATUS RESTARTS AGE cluster-logging-operator-668846c54c-6477k 1/1 Running 0 15h instance-2wp6g 1/1 Running 0 7s instance-9jms7 1/1 Running 0 7s instance-n9mjs 1/1 Running 0 7s instance-qr9hs 1/1 Running 0 6s instance-vhcx2 1/1 Running 0 6s lokistack-ltetz-compactor-0 1/1 Running 0 15h lokistack-ltetz-distributor-7f55b6dfdc-pz9pd 1/1 Running 0 15h lokistack-ltetz-gateway-77979b95f7-gnb74 2/2 Running 0 15h lokistack-ltetz-gateway-77979b95f7-lnxln 2/2 Running 0 15h lokistack-ltetz-index-gateway-0 1/1 Running 0 15h lokistack-ltetz-ingester-0 1/1 Running 0 15h lokistack-ltetz-querier-6bcfb55494-jjn7c 1/1 Running 0 15h lokistack-ltetz-query-frontend-7fb4bdcb58-pmj5t 1/1 Running 0 15h lokistack-ltetz-ruler-0 1/1 Running 0 15h [root@rdr-jeev-415log-bastion-0 ~]# ``` GCP logging: Application logs: ``` { "insertId": "1doxk10fchgm2o", "jsonPayload": { "@timestamp": "2025-05-07T07:07:01.795356251Z", "hostname": "worker-0", "kubernetes": { "annotations": { "k8s.ovn.org/pod-networks": "{\"default\":{\"ip_addresses\":[\"10.131.0.76/23\"],\"mac_address\":\"0a:58:0a:83:00:4c\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.76/23\",\"gateway_ip\":\"10.131.0.1\"}}", "k8s.v1.cni.cncf.io/network-status": "[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.76\"\n ],\n \"mac\": \"0a:58:0a:83:00:4c\",\n \"default\": true,\n \"dns\": {}\n}]", "openshift.io/scc": "restricted-v2", "seccomp.security.alpha.kubernetes.io/pod": "runtime/default" }, "container_id": "cri-o://5ebe6cfeb054b528e3c048d15d7a12e40e474e1e3fb95b417a4478270fddbfb6", "container_image": "quay.io/pravin_dsilva/centos:logtest", "container_image_id": "quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912", "container_name": "centos-logtest", "labels": { "run": "centos-logtest", "test": "centos-logtest" }, "namespace_id": "e2166d8f-cd66-4fec-b7fe-df5234e1507a", "namespace_labels": { "kubernetes_io_metadata_name": "test-logging", "pod-security_kubernetes_io_audit": "restricted", "pod-security_kubernetes_io_audit-version": "v1.24", "pod-security_kubernetes_io_warn": "restricted", "pod-security_kubernetes_io_warn-version": "v1.24" }, "namespace_name": "test-logging", "pod_id": "c603b0ef-f979-4a5f-9e69-9ba0d44d3808", "pod_ip": "10.131.0.76", "pod_name": "centos-logtest-kfr6v", "pod_owner": "ReplicationController/centos-logtest" }, "log_source": "container", "log_type": "application", "message": "{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }", "openshift": { "cluster_id": "ae9a9f3f-409f-486a-9228-008db9788e0c", "labels": { "label": "6.0.8_415-app" }, "sequence": 1.74660162185485e+18 } }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-app", "receiveTimestamp": "2025-05-07T07:07:02.872561526Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "timestamp": "2025-05-07T07:07:02.872561526Z" }, { "insertId": "1nz7wbbfd8gglu", "jsonPayload": { "@timestamp": "2025-05-07T07:07:00.793821540Z", "hostname": "worker-0", "kubernetes": { "annotations": { "k8s.ovn.org/pod-networks": "{\"default\":{\"ip_addresses\":[\"10.131.0.76/23\"],\"mac_address\":\"0a:58:0a:83:00:4c\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.76/23\",\"gateway_ip\":\"10.131.0.1\"}}", "k8s.v1.cni.cncf.io/network-status": "[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.76\"\n ],\n \"mac\": \"0a:58:0a:83:00:4c\",\n \"default\": true,\n \"dns\": {}\n}]", "openshift.io/scc": "restricted-v2", "seccomp.security.alpha.kubernetes.io/pod": "runtime/default" }, "container_id": "cri-o://5ebe6cfeb054b528e3c048d15d7a12e40e474e1e3fb95b417a4478270fddbfb6", "container_image": "quay.io/pravin_dsilva/centos:logtest", "container_image_id": "quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912", "container_name": "centos-logtest", "labels": { "run": "centos-logtest", "test": "centos-logtest" }, "namespace_id": "e2166d8f-cd66-4fec-b7fe-df5234e1507a", "namespace_labels": { "kubernetes_io_metadata_name": "test-logging", "pod-security_kubernetes_io_audit": "restricted", "pod-security_kubernetes_io_audit-version": "v1.24", "pod-security_kubernetes_io_warn": "restricted", "pod-security_kubernetes_io_warn-version": "v1.24" }, "namespace_name": "test-logging", "pod_id": "c603b0ef-f979-4a5f-9e69-9ba0d44d3808", "pod_ip": "10.131.0.76", "pod_name": "centos-logtest-kfr6v", "pod_owner": "ReplicationController/centos-logtest" }, "log_source": "container", "log_type": "application", "message": "{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }", "openshift": { "cluster_id": "ae9a9f3f-409f-486a-9228-008db9788e0c", "labels": { "label": "6.0.8_415-app" }, "sequence": 1.746601620822202e+18 } }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-app", "receiveTimestamp": "2025-05-07T07:07:01.318003416Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "timestamp": "2025-05-07T07:07:01.318003416Z" } ] ``` Audit logs: ``` { "insertId": "3xputyfca1h16", "jsonPayload": { "@timestamp": "2025-05-07T07:07:42.027716411Z", "annotations": { "authorization.k8s.io/decision": "allow", "authorization.k8s.io/reason": "RBAC: allowed by RoleBinding \"console-operator/openshift-console\" of Role \"console-operator\" to ServiceAccount \"console-operator/openshift-console-operator\"" }, "apiVersion": "audit.k8s.io/v1", "auditID": "b858a62e-8cd9-46fb-aee7-d9f4205cc3f3", "hostname": "master-2", "k8s_audit_level": "Metadata", "kind": "Event", "log_source": "kubeAPI", "log_type": "audit", "objectRef": { "apiVersion": "v1", "name": "trusted-ca-bundle", "namespace": "openshift-console", "resource": "configmaps" }, "openshift": { "cluster_id": "ae9a9f3f-409f-486a-9228-008db9788e0c", "labels": { "label": "6.0.8_415-audit" }, "sequence": 1.746601662028339e+18 }, "requestReceivedTimestamp": "2025-05-07T07:07:41.915714Z", "requestURI": "/api/v1/namespaces/openshift-console/configmaps/trusted-ca-bundle", "responseStatus": { "code": 200, "metadata": {} }, "sourceIPs": [ "10.20.177.64" ], "stage": "ResponseComplete", "stageTimestamp": "2025-05-07T07:07:41.926044Z", "user": { "extra": { "authentication.kubernetes.io/pod-name": [ "console-operator-7fd447cff9-mk6sd" ], "authentication.kubernetes.io/pod-uid": [ "1264cbee-766a-43fd-b9a7-7aea9feb0383" ] }, "groups": [ "system:serviceaccounts", "system:serviceaccounts:openshift-console-operator", "system:authenticated" ], "uid": "dd704e3e-aab6-4f34-8efd-d0bf8a148c71", "username": "system:serviceaccount:openshift-console-operator:console-operator" }, "userAgent": "console/v0.0.0 (linux/ppc64le) kubernetes/$Format", "verb": "get" }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-audit", "receiveTimestamp": "2025-05-07T07:07:42.540617795Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "master-2", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-05-07T07:07:42.540617795Z" }, { "insertId": "3xputyfca1h15", "jsonPayload": { "@timestamp": "2025-05-07T07:07:42.027709083Z", "annotations": { "authorization.k8s.io/decision": "allow", "authorization.k8s.io/reason": "RBAC: allowed by RoleBinding \"console-operator/openshift-console\" of Role \"console-operator\" to ServiceAccount \"console-operator/openshift-console-operator\"" }, "apiVersion": "audit.k8s.io/v1", "auditID": "70b3ed65-94fa-4485-88db-58a21cea61e5", "hostname": "master-2", "k8s_audit_level": "Metadata", "kind": "Event", "log_source": "kubeAPI", "log_type": "audit", "objectRef": { "apiGroup": "route.openshift.io", "apiVersion": "v1", "name": "console", "namespace": "openshift-console", "resource": "routes" }, "openshift": { "cluster_id": "ae9a9f3f-409f-486a-9228-008db9788e0c", "labels": { "label": "6.0.8_415-audit" }, "sequence": 1.746601662028301e+18 }, "requestReceivedTimestamp": "2025-05-07T07:07:41.908578Z", "requestURI": "/apis/route.openshift.io/v1/namespaces/openshift-console/routes/console", "responseStatus": { "code": 200, "metadata": {} }, "sourceIPs": [ "10.20.177.64" ], "stage": "ResponseComplete", "stageTimestamp": "2025-05-07T07:07:41.914904Z", "user": { "extra": { "authentication.kubernetes.io/pod-name": [ "console-operator-7fd447cff9-mk6sd" ], "authentication.kubernetes.io/pod-uid": [ "1264cbee-766a-43fd-b9a7-7aea9feb0383" ] }, "groups": [ "system:serviceaccounts", "system:serviceaccounts:openshift-console-operator", "system:authenticated" ], "uid": "dd704e3e-aab6-4f34-8efd-d0bf8a148c71", "username": "system:serviceaccount:openshift-console-operator:console-operator" }, "userAgent": "console/v0.0.0 (linux/ppc64le) kubernetes/$Format", "verb": "get" }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-audit", "receiveTimestamp": "2025-05-07T07:07:42.540617795Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "master-2", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-05-07T07:07:42.540617795Z" } ] ``` Infrastructure logs: ``` { "insertId": "46bay3fdifc88", "jsonPayload": { "@timestamp": "2025-05-07T07:11:20.843894503Z", "hostname": "worker-0", "kubernetes": { "annotations": { "imageregistry.operator.openshift.io/dependencies-checksum": "sha256:dc4da576459e75e60ff91c10184450e3aade66e2c095ff3926e66a0a0495e72c", "k8s.ovn.org/pod-networks": "{\"default\":{\"ip_addresses\":[\"10.131.0.80/23\"],\"mac_address\":\"0a:58:0a:83:00:50\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.80/23\",\"gateway_ip\":\"10.131.0.1\"}}", "k8s.v1.cni.cncf.io/network-status": "[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.80\"\n ],\n \"mac\": \"0a:58:0a:83:00:50\",\n \"default\": true,\n \"dns\": {}\n}]", "openshift.io/required-scc": "restricted-v2", "openshift.io/scc": "restricted-v2", "seccomp.security.alpha.kubernetes.io/pod": "runtime/default" }, "container_id": "cri-o://c648c3fb031adaba294f71c09787dd3246eb240c6d6e73aa70d42fb08dc2da10", "container_image": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:28d53ac4d1793b0d449f7e90f813711d25802b0296dcfb57021e8a9a91da89cf", "container_image_id": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:28d53ac4d1793b0d449f7e90f813711d25802b0296dcfb57021e8a9a91da89cf", "container_name": "registry", "labels": { "docker-registry": "default", "pod-template-hash": "9cc6bc6c6" }, "namespace_id": "4e15b04f-023c-47f3-8303-363e8ffbe86f", "namespace_labels": { "kubernetes_io_metadata_name": "openshift-image-registry", "olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5": "", "openshift_io_cluster-monitoring": "true", "pod-security_kubernetes_io_audit": "privileged", "pod-security_kubernetes_io_enforce": "privileged", "pod-security_kubernetes_io_warn": "privileged" }, "namespace_name": "openshift-image-registry", "pod_id": "e0a2fc7e-abfa-4375-a803-73f5957819fe", "pod_ip": "10.131.0.80", "pod_name": "image-registry-9cc6bc6c6-hpznp", "pod_owner": "ReplicaSet/image-registry-9cc6bc6c6" }, "log_source": "container", "log_type": "infrastructure", "message": "time=\"2025-05-07T07:11:20.843852843Z\" level=info msg=response go.version=\"go1.20.12 X:strictfipsruntime\" http.request.host=\"10.131.0.80:5000\" http.request.id=cdee041b-69ad-4a68-a26f-095c55a86132 http.request.method=GET http.request.remoteaddr=\"10.128.3.115:36918\" http.request.uri=/extensions/v2/metrics http.request.useragent=Prometheus/2.48.0 http.response.contenttype=\"text/plain; version=0.0.4; charset=utf-8\" http.response.duration=13.052797ms http.response.status=200 http.response.written=2132", "openshift": { "cluster_id": "ae9a9f3f-409f-486a-9228-008db9788e0c", "labels": { "label": "6.0.8_415-infra" }, "sequence": 1.7466018809845555e+18 } }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-infra", "receiveTimestamp": "2025-05-07T07:11:22.014674872Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-05-07T07:11:22.014674872Z" }, { "insertId": "46bay3fdifc87", "jsonPayload": { "@timestamp": "2025-05-07T07:11:20.843894503Z", "hostname": "worker-0", "kubernetes": { "annotations": { "imageregistry.operator.openshift.io/dependencies-checksum": "sha256:dc4da576459e75e60ff91c10184450e3aade66e2c095ff3926e66a0a0495e72c", "k8s.ovn.org/pod-networks": "{\"default\":{\"ip_addresses\":[\"10.131.0.80/23\"],\"mac_address\":\"0a:58:0a:83:00:50\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.80/23\",\"gateway_ip\":\"10.131.0.1\"}}", "k8s.v1.cni.cncf.io/network-status": "[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.80\"\n ],\n \"mac\": \"0a:58:0a:83:00:50\",\n \"default\": true,\n \"dns\": {}\n}]", "openshift.io/required-scc": "restricted-v2", "openshift.io/scc": "restricted-v2", "seccomp.security.alpha.kubernetes.io/pod": "runtime/default" }, "container_id": "cri-o://c648c3fb031adaba294f71c09787dd3246eb240c6d6e73aa70d42fb08dc2da10", "container_image": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:28d53ac4d1793b0d449f7e90f813711d25802b0296dcfb57021e8a9a91da89cf", "container_image_id": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:28d53ac4d1793b0d449f7e90f813711d25802b0296dcfb57021e8a9a91da89cf", "container_name": "registry", "labels": { "docker-registry": "default", "pod-template-hash": "9cc6bc6c6" }, "namespace_id": "4e15b04f-023c-47f3-8303-363e8ffbe86f", "namespace_labels": { "kubernetes_io_metadata_name": "openshift-image-registry", "olm_operatorgroup_uid_3fd1a0b8-266e-43cf-bb67-3d5e4bbdf2a5": "", "openshift_io_cluster-monitoring": "true", "pod-security_kubernetes_io_audit": "privileged", "pod-security_kubernetes_io_enforce": "privileged", "pod-security_kubernetes_io_warn": "privileged" }, "namespace_name": "openshift-image-registry", "pod_id": "e0a2fc7e-abfa-4375-a803-73f5957819fe", "pod_ip": "10.131.0.80", "pod_name": "image-registry-9cc6bc6c6-hpznp", "pod_owner": "ReplicaSet/image-registry-9cc6bc6c6" }, "log_source": "container", "log_type": "infrastructure", "message": "time=\"2025-05-07T07:11:20.843783272Z\" level=info msg=\"response completed\" go.version=\"go1.20.12 X:strictfipsruntime\" http.request.host=\"10.131.0.80:5000\" http.request.id=95289e67-5401-483a-8233-0b4020209fc2 http.request.method=GET http.request.remoteaddr=\"10.128.3.115:36918\" http.request.uri=/extensions/v2/metrics http.request.useragent=Prometheus/2.48.0 http.response.contenttype=\"text/plain; version=0.0.4; charset=utf-8\" http.response.duration=12.859857ms http.response.status=200 http.response.written=2132 openshift.auth.user=\"system:serviceaccount:openshift-monitoring:prometheus-k8s\" openshift.auth.userid=ba1909e8-55ec-46e0-968f-bb65b6ee795e", "openshift": { "cluster_id": "ae9a9f3f-409f-486a-9228-008db9788e0c", "labels": { "label": "6.0.8_415-infra" }, "sequence": 1.746601880984524e+18 } }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-infra", "receiveTimestamp": "2025-05-07T07:11:22.014674872Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-05-07T07:11:22.014674872Z" } ] ```