Cluster logging 6.0.7 on OCP 4.15 with GCP CSV and Pods: ``` [root@rdr-man-log-415-bastion-0 ~]# oc get csv -n openshift-logging NAME DISPLAY VERSION REPLACES PHASE cluster-logging.v6.0.7 Red Hat OpenShift Logging 6.0.7 cluster-logging.v6.0.6 Succeeded loki-operator.v6.0.7 Loki Operator 6.0.7 loki-operator.v6.0.6 Succeeded [root@rdr-man-log-415-bastion-0 ~]# oc get pods -n openshift-logging NAME READY STATUS RESTARTS AGE cluster-logging-operator-d66b6757b-dp6xx 1/1 Running 0 173m instance-44mcd 1/1 Running 0 40s instance-57kft 1/1 Running 0 40s instance-blvmv 1/1 Running 0 40s instance-fq66n 1/1 Running 0 40s instance-mvgph 1/1 Running 0 40s lokistack-almqn-compactor-0 1/1 Running 0 172m lokistack-almqn-distributor-599d5dfc4c-d9pwz 1/1 Running 0 172m lokistack-almqn-gateway-6f4976c89c-64nx9 2/2 Running 1 (172m ago) 172m lokistack-almqn-gateway-6f4976c89c-x8jr8 2/2 Running 1 (172m ago) 172m lokistack-almqn-index-gateway-0 1/1 Running 0 172m lokistack-almqn-ingester-0 1/1 Running 0 172m lokistack-almqn-querier-6945647454-nwptn 1/1 Running 0 172m lokistack-almqn-query-frontend-5d8477fc9b-qzgr5 1/1 Running 0 172m lokistack-almqn-ruler-0 1/1 Running 0 172m [root@rdr-man-log-415-bastion-0 ~]# ``` Cluster logging: CloudWatch: Application logs: ``` { "insertId": "17knbjgfyveogt", "jsonPayload": { "@timestamp": "2025-04-16T11:58:31.964291884Z", "hostname": "worker-1", "kubernetes": { "annotations": { "k8s.ovn.org/pod-networks": "{\"default\":{\"ip_addresses\":[\"10.131.0.143/23\"],\"mac_address\":\"0a:58:0a:83:00:8f\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.143/23\",\"gateway_ip\":\"10.131.0.1\"}}", "k8s.v1.cni.cncf.io/network-status": "[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.143\"\n ],\n \"mac\": \"0a:58:0a:83:00:8f\",\n \"default\": true,\n \"dns\": {}\n}]", "openshift.io/scc": "restricted-v2", "seccomp.security.alpha.kubernetes.io/pod": "runtime/default" }, "container_id": "cri-o://1432e6f7f911cefc690e46c302089bf6416fc9972cab0effeb2909f2ef2ed492", "container_image": "quay.io/pravin_dsilva/centos:logtest", "container_image_id": "quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912", "container_name": "centos-logtest", "labels": { "run": "centos-logtest", "test": "centos-logtest" }, "namespace_id": "43d4ffb8-71b1-4110-96e6-100376c989e8", "namespace_labels": { "kubernetes_io_metadata_name": "test-logging", "pod-security_kubernetes_io_audit": "restricted", "pod-security_kubernetes_io_audit-version": "v1.24", "pod-security_kubernetes_io_warn": "restricted", "pod-security_kubernetes_io_warn-version": "v1.24" }, "namespace_name": "test-logging", "pod_id": "b94ff1c1-5294-4ca4-a2ff-a1403368c9e7", "pod_ip": "10.131.0.143", "pod_name": "centos-logtest-9dvww", "pod_owner": "ReplicationController/centos-logtest" }, "log_source": "container", "log_type": "application", "message": "{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }", "openshift": { "cluster_id": "f3ed82e8-831f-45fe-a973-d869e9d46576", "labels": { "label": "6.0_417-app" }, "sequence": 1.7448047120480044e+18 } }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-app", "receiveTimestamp": "2025-04-16T11:58:33.066286239Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-1", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "timestamp": "2025-04-16T11:58:33.066286239Z" }, { "insertId": "1hajllafzmdt88", "jsonPayload": { "@timestamp": "2025-04-16T11:58:30.962893391Z", "hostname": "worker-1", "kubernetes": { "annotations": { "k8s.ovn.org/pod-networks": "{\"default\":{\"ip_addresses\":[\"10.131.0.143/23\"],\"mac_address\":\"0a:58:0a:83:00:8f\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"169.254.169.5/32\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.143/23\",\"gateway_ip\":\"10.131.0.1\"}}", "k8s.v1.cni.cncf.io/network-status": "[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.143\"\n ],\n \"mac\": \"0a:58:0a:83:00:8f\",\n \"default\": true,\n \"dns\": {}\n}]", "openshift.io/scc": "restricted-v2", "seccomp.security.alpha.kubernetes.io/pod": "runtime/default" }, "container_id": "cri-o://1432e6f7f911cefc690e46c302089bf6416fc9972cab0effeb2909f2ef2ed492", "container_image": "quay.io/pravin_dsilva/centos:logtest", "container_image_id": "quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912", "container_name": "centos-logtest", "labels": { "run": "centos-logtest", "test": "centos-logtest" }, "namespace_id": "43d4ffb8-71b1-4110-96e6-100376c989e8", "namespace_labels": { "kubernetes_io_metadata_name": "test-logging", "pod-security_kubernetes_io_audit": "restricted", "pod-security_kubernetes_io_audit-version": "v1.24", "pod-security_kubernetes_io_warn": "restricted", "pod-security_kubernetes_io_warn-version": "v1.24" }, "namespace_name": "test-logging", "pod_id": "b94ff1c1-5294-4ca4-a2ff-a1403368c9e7", "pod_ip": "10.131.0.143", "pod_name": "centos-logtest-9dvww", "pod_owner": "ReplicationController/centos-logtest" }, "log_source": "container", "log_type": "application", "message": "{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }", "openshift": { "cluster_id": "f3ed82e8-831f-45fe-a973-d869e9d46576", "labels": { "label": "6.0_417-app" }, "sequence": 1.7448047110137298e+18 } }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-app", "receiveTimestamp": "2025-04-16T11:58:32.032600646Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-1", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "timestamp": "2025-04-16T11:58:32.032600646Z" } ] ``` Audit logs: ``` { "insertId": "1xyi8qkfzredfw", "jsonPayload": { "@timestamp": "2025-04-16T12:07:26.988416241Z", "annotations": { "authorization.k8s.io/decision": "allow", "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"openshift-state-metrics\" of ClusterRole \"openshift-state-metrics\" to ServiceAccount \"openshift-state-metrics/openshift-monitoring\"" }, "apiVersion": "audit.k8s.io/v1", "auditID": "9b70bb81-d90f-432c-96e9-9508f044e9da", "hostname": "master-0", "kind": "Event", "log_source": "openshiftAPI", "log_type": "audit", "objectRef": { "apiGroup": "build.openshift.io", "apiVersion": "v1", "resource": "buildconfigs" }, "openshift": { "cluster_id": "f3ed82e8-831f-45fe-a973-d869e9d46576", "labels": { "label": "6.0_417-audit" }, "sequence": 1.744805246988998e+18 }, "openshift_audit_level": "Metadata", "requestReceivedTimestamp": "2025-04-16T11:58:46.531509Z", "requestURI": "/apis/build.openshift.io/v1/buildconfigs?allowWatchBookmarks=true&resourceVersion=2310608&timeout=8m40s&timeoutSeconds=520&watch=true", "responseStatus": { "code": 200, "metadata": {} }, "sourceIPs": [ "10.20.177.136", "10.128.0.2" ], "stage": "ResponseComplete", "stageTimestamp": "2025-04-16T12:07:26.556065Z", "user": { "extra": { "authentication.kubernetes.io/pod-name": [ "openshift-state-metrics-7f5dccd865-qqvqd" ], "authentication.kubernetes.io/pod-uid": [ "20e6b5ae-1f83-4214-bbff-e9ea974e0577" ] }, "groups": [ "system:serviceaccounts", "system:serviceaccounts:openshift-monitoring", "system:authenticated" ], "username": "system:serviceaccount:openshift-monitoring:openshift-state-metrics" }, "userAgent": "openshift-state-metrics/UNKNOWN (linux/ppc64le) kube-state-metrics/UNKNOWN", "verb": "watch" }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-audit", "receiveTimestamp": "2025-04-16T12:07:27.504691873Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "master-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-04-16T12:07:27.504691873Z" }, { "insertId": "1xyi8qkfzredfv", "jsonPayload": { "@timestamp": "2025-04-16T12:07:26.988409440Z", "annotations": { "authorization.k8s.io/decision": "allow", "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"cluster-status-binding\" of ClusterRole \"cluster-status\" to Group \"system:authenticated\"" }, "apiVersion": "audit.k8s.io/v1", "auditID": "c396eb04-aa7e-4c73-a9e6-8146564fd2a4", "hostname": "master-0", "kind": "Event", "log_source": "openshiftAPI", "log_type": "audit", "openshift": { "cluster_id": "f3ed82e8-831f-45fe-a973-d869e9d46576", "labels": { "label": "6.0_417-audit" }, "sequence": 1.7448052469889595e+18 }, "openshift_audit_level": "Metadata", "requestReceivedTimestamp": "2025-04-16T12:07:22.544573Z", "requestURI": "/openapi/v2", "responseStatus": { "code": 304, "metadata": {} }, "sourceIPs": [ "10.130.0.2" ], "stage": "ResponseComplete", "stageTimestamp": "2025-04-16T12:07:22.545336Z", "user": { "groups": [ "system:authenticated" ], "username": "system:aggregator" }, "verb": "get" }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-audit", "receiveTimestamp": "2025-04-16T12:07:27.504691873Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "master-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-04-16T12:07:27.504691873Z" } ] ``` Infrastructure logs: ``` { "insertId": "171x647g12d7s6b", "jsonPayload": { "@timestamp": "2025-04-16T12:08:11.238043898Z", "hostname": "master-1", "kubernetes": { "container_id": "cri-o://2758d2c2a6525bb26f1e84a3bf3942dc09e31e64e598665ca5b66a26dbf9c8bc", "container_image": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d605557ebf11f602dc90289889c4567cdfec487c0f3e968e3d349a1f2cedffa2", "container_image_id": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d605557ebf11f602dc90289889c4567cdfec487c0f3e968e3d349a1f2cedffa2", "container_name": "network-operator", "labels": { "name": "network-operator", "pod-template-hash": "66dd77b46d" }, "namespace_id": "dd4c3611-ce62-4b38-b7ac-d59b7bc4360a", "namespace_labels": { "kubernetes_io_metadata_name": "openshift-network-operator", "name": "openshift-network-operator", "olm_operatorgroup_uid_2a1082ff-0097-4440-a25e-b107a7a3319d": "", "openshift_io_cluster-monitoring": "true", "openshift_io_run-level": "0", "pod-security_kubernetes_io_audit": "privileged", "pod-security_kubernetes_io_enforce": "privileged", "pod-security_kubernetes_io_warn": "privileged" }, "namespace_name": "openshift-network-operator", "pod_id": "719f4d0a-8339-4a0a-9beb-90b328914fe8", "pod_ip": "10.20.177.61", "pod_name": "network-operator-66dd77b46d-vwvvm", "pod_owner": "ReplicaSet/network-operator-66dd77b46d" }, "log_source": "container", "log_type": "infrastructure", "message": "I0416 12:08:11.237965 1 log.go:194] Apply / Create of (rbac.authorization.k8s.io/v1, Kind=ClusterRole) /multus-admission-controller-webhook was successful", "openshift": { "cluster_id": "f3ed82e8-831f-45fe-a973-d869e9d46576", "labels": { "label": "6.0_417-infra" }, "sequence": 1.744805291242187e+18 } }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-infra", "receiveTimestamp": "2025-04-16T12:08:11.475310049Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "master-1", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-04-16T12:08:11.475310049Z" }, { "insertId": "171x647g12d7s6a", "jsonPayload": { "@timestamp": "2025-04-16T12:08:10.836550168Z", "hostname": "master-1", "kubernetes": { "container_id": "cri-o://2758d2c2a6525bb26f1e84a3bf3942dc09e31e64e598665ca5b66a26dbf9c8bc", "container_image": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d605557ebf11f602dc90289889c4567cdfec487c0f3e968e3d349a1f2cedffa2", "container_image_id": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d605557ebf11f602dc90289889c4567cdfec487c0f3e968e3d349a1f2cedffa2", "container_name": "network-operator", "labels": { "name": "network-operator", "pod-template-hash": "66dd77b46d" }, "namespace_id": "dd4c3611-ce62-4b38-b7ac-d59b7bc4360a", "namespace_labels": { "kubernetes_io_metadata_name": "openshift-network-operator", "name": "openshift-network-operator", "olm_operatorgroup_uid_2a1082ff-0097-4440-a25e-b107a7a3319d": "", "openshift_io_cluster-monitoring": "true", "openshift_io_run-level": "0", "pod-security_kubernetes_io_audit": "privileged", "pod-security_kubernetes_io_enforce": "privileged", "pod-security_kubernetes_io_warn": "privileged" }, "namespace_name": "openshift-network-operator", "pod_id": "719f4d0a-8339-4a0a-9beb-90b328914fe8", "pod_ip": "10.20.177.61", "pod_name": "network-operator-66dd77b46d-vwvvm", "pod_owner": "ReplicaSet/network-operator-66dd77b46d" }, "log_source": "container", "log_type": "infrastructure", "message": "I0416 12:08:10.836538 1 log.go:194] reconciling (rbac.authorization.k8s.io/v1, Kind=ClusterRole) /multus-admission-controller-webhook", "openshift": { "cluster_id": "f3ed82e8-831f-45fe-a973-d869e9d46576", "labels": { "label": "6.0_417-infra" }, "sequence": 1.7448052909785777e+18 } }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-infra", "receiveTimestamp": "2025-04-16T12:08:11.475310049Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "master-1", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-04-16T12:08:11.475310049Z" } ] ```