GCP on ocp 4.14 with 5.8.19 CSV and Pods: ``` [root@rdr-man-414log-bastion-0 ~]# oc get csv -n openshift-logging NAME DISPLAY VERSION REPLACES PHASE cluster-logging.v5.8.19 Red Hat OpenShift Logging 5.8.19 cluster-logging.v5.8.18 Succeeded cluster-observability-operator.v1.1.0 Cluster Observability Operator 1.1.0 cluster-observability-operator.v1.0.0 Succeeded elasticsearch-operator.v5.8.19 OpenShift Elasticsearch Operator 5.8.19 elasticsearch-operator.v5.8.18 Succeeded loki-operator.v5.8.19 Loki Operator 5.8.19 loki-operator.v5.8.18 Succeeded [root@rdr-man-414log-bastion-0 ~]# oc get pods -n openshift-logging NAME READY STATUS RESTARTS AGE cluster-logging-operator-9f4795575-km4bb 1/1 Running 0 33h collector-6fdrf 1/1 Running 0 15m collector-gvl4h 1/1 Running 0 15m collector-h946j 1/1 Running 0 15m collector-lpx25 1/1 Running 0 15m collector-vfv9p 1/1 Running 0 15m elasticsearch-cdm-5rw0thq6-1-688549bd7d-rv9cw 2/2 Running 0 36m elasticsearch-cdm-5rw0thq6-2-66f595dbf5-x75p6 2/2 Running 0 36m elasticsearch-cdm-5rw0thq6-3-8694699764-7npsb 2/2 Running 0 36m elasticsearch-im-app-29058825-24qpq 0/1 Completed 0 9m26s elasticsearch-im-audit-29058825-sgpqm 0/1 Completed 0 9m26s elasticsearch-im-infra-29058825-79qsz 0/1 Completed 0 9m26s kibana-55c4848997-hfrhj 2/2 Running 0 36m [root@rdr-man-414log-bastion-0 ~]# ``` Application logs: ``` { "insertId": "1dyyq3zf3l75e6", "jsonPayload": { "@timestamp": "2025-04-01T17:40:27.748109600Z", "file": "/var/log/pods/test-logging_centos-logtest-76qb9_5f049a1e-0586-4493-98a1-9329b8cf1d7b/centos-logtest/0.log", "hostname": "worker-1", "kubernetes": { "annotations": { "k8s.ovn.org/pod-networks": "{\"default\":{\"ip_addresses\":[\"10.131.0.174/23\"],\"mac_address\":\"0a:58:0a:83:00:ae\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.174/23\",\"gateway_ip\":\"10.131.0.1\"}}", "k8s.v1.cni.cncf.io/network-status": "[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.174\"\n ],\n \"mac\": \"0a:58:0a:83:00:ae\",\n \"default\": true,\n \"dns\": {}\n}]", "openshift.io/scc": "restricted-v2", "seccomp.security.alpha.kubernetes.io/pod": "runtime/default" }, "container_id": "cri-o://728df538617fa22d57cac9d28b79079398171cdd1f24897aec60fe0e43a8135e", "container_image": "quay.io/pravin_dsilva/centos:logtest", "container_name": "centos-logtest", "labels": { "run": "centos-logtest", "test": "centos-logtest" }, "namespace_id": "5a1de092-661d-4860-b096-cf853ef15b38", "namespace_labels": { "kubernetes_io_metadata_name": "test-logging", "pod-security_kubernetes_io_audit": "restricted", "pod-security_kubernetes_io_audit-version": "v1.24", "pod-security_kubernetes_io_warn": "restricted", "pod-security_kubernetes_io_warn-version": "v1.24" }, "namespace_name": "test-logging", "pod_id": "5f049a1e-0586-4493-98a1-9329b8cf1d7b", "pod_ip": "10.131.0.174", "pod_name": "centos-logtest-76qb9", "pod_owner": "ReplicationController/centos-logtest" }, "log_type": "application", "message": "2025-04-01 17:40:27,747 - SVTLogger - INFO - centos-logtest-76qb9 : 367559 : vRbDfq1QU PJPUR4Hdp hH3OaPxCv bWmWEHEWa YjNzvWMhq nIaGsTX0O JdCXdYart RvxtSNW8m o19v45lVa yfjE1b5x3 or29hvNAM KpaFnINVU 6SWegLOIf SLSXVzt15 BNMU5DKjW VtclrkjmB hYbKkQHc1 GVWEqRSFL MM6y8mSzO d9wmAUElI ", "openshift": { "cluster_id": "657154e1-ef9c-426e-8079-d58eaee0a17b", "sequence": 191 } }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-app", "receiveTimestamp": "2025-04-01T17:40:28.778868464Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-1", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-04-01T17:40:28.778868464Z" }, { "insertId": "1bq8lzef3i2y8q", "jsonPayload": { "@timestamp": "2025-04-01T17:40:25.745109139Z", "file": "/var/log/pods/exclude-app_centos-logtest-gjd47_e522a963-f9e7-40e6-a502-fcaa5e58614f/centos-logtest/0.log", "hostname": "worker-1", "kubernetes": { "annotations": { "k8s.ovn.org/pod-networks": "{\"default\":{\"ip_addresses\":[\"10.131.0.175/23\"],\"mac_address\":\"0a:58:0a:83:00:af\",\"gateway_ips\":[\"10.131.0.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.131.0.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.131.0.1\"}],\"ip_address\":\"10.131.0.175/23\",\"gateway_ip\":\"10.131.0.1\"}}", "k8s.v1.cni.cncf.io/network-status": "[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.131.0.175\"\n ],\n \"mac\": \"0a:58:0a:83:00:af\",\n \"default\": true,\n \"dns\": {}\n}]", "openshift.io/scc": "restricted-v2", "seccomp.security.alpha.kubernetes.io/pod": "runtime/default" }, "container_id": "cri-o://1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00", "container_image": "quay.io/pravin_dsilva/centos:logtest", "container_name": "centos-logtest", "labels": { "run": "centos-logtest", "test": "centos-logtest" }, "namespace_id": "4566a766-d384-4fb9-8430-1c31bd2242b5", "namespace_labels": { "kubernetes_io_metadata_name": "exclude-app", "pod-security_kubernetes_io_audit": "restricted", "pod-security_kubernetes_io_audit-version": "v1.24", "pod-security_kubernetes_io_warn": "restricted", "pod-security_kubernetes_io_warn-version": "v1.24" }, "namespace_name": "exclude-app", "pod_id": "e522a963-f9e7-40e6-a502-fcaa5e58614f", "pod_ip": "10.131.0.175", "pod_name": "centos-logtest-gjd47", "pod_owner": "ReplicationController/centos-logtest" }, "log_type": "application", "message": "2025-04-01 17:40:25,744 - SVTLogger - INFO - centos-logtest-gjd47 : 367567 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu ", "openshift": { "cluster_id": "657154e1-ef9c-426e-8079-d58eaee0a17b", "sequence": 186 } }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-app", "receiveTimestamp": "2025-04-01T17:40:26.394225866Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-1", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-04-01T17:40:26.394225866Z" } ] ``` Audit logs: ``` { "insertId": "1kdp9wzf39vuvr", "jsonPayload": { "@timestamp": "2025-04-01T17:52:37.036391528Z", "annotations": { "authorization.k8s.io/decision": "allow", "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"elasticsearch-operator.v5.8.19-66468cf8d7\" of ClusterRole \"elasticsearch-operator.v5.8.19-66468cf8d7\" to ServiceAccount \"elasticsearch-operator/openshift-operators-redhat\"" }, "apiVersion": "audit.k8s.io/v1", "auditID": "edb2e784-f62c-44eb-84fc-3df78bb1e917", "file": "/var/log/kube-apiserver/audit.log", "hostname": "master-2", "k8s_audit_level": "Metadata", "kind": "Event", "log_type": "audit", "objectRef": { "apiGroup": "logging.openshift.io", "apiVersion": "v1", "name": "elasticsearch", "namespace": "openshift-logging", "resource": "elasticsearches", "resourceVersion": "5450588", "subresource": "status", "uid": "7218b883-2d52-49be-9912-305da2d80130" }, "openshift": { "cluster_id": "657154e1-ef9c-426e-8079-d58eaee0a17b", "sequence": 63286 }, "requestReceivedTimestamp": "2025-04-01T17:52:36.929454Z", "requestURI": "/apis/logging.openshift.io/v1/namespaces/openshift-logging/elasticsearches/elasticsearch/status", "responseStatus": { "code": 200, "metadata": {} }, "sourceIPs": [ "10.20.184.84" ], "source_type": "file", "stage": "ResponseComplete", "stageTimestamp": "2025-04-01T17:52:36.969258Z", "tag": ".k8s-audit.log", "user": { "extra": { "authentication.kubernetes.io/pod-name": [ "elasticsearch-operator-b78bc8575-2z9cz" ], "authentication.kubernetes.io/pod-uid": [ "d7de42e8-9cdd-4375-b923-f9450eb3f319" ] }, "groups": [ "system:serviceaccounts", "system:serviceaccounts:openshift-operators-redhat", "system:authenticated" ], "uid": "3ca20cad-7d98-4e03-a5e8-bde08b6751a4", "username": "system:serviceaccount:openshift-operators-redhat:elasticsearch-operator" }, "userAgent": "elasticsearch-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format", "verb": "update" }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-audit", "receiveTimestamp": "2025-04-01T17:52:37.723290209Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "master-2", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "timestamp": "2025-04-01T17:52:37.723290209Z" }, { "insertId": "1kdp9wzf39vuvq", "jsonPayload": { "@timestamp": "2025-04-01T17:52:36.707157268Z", "annotations": { "authorization.k8s.io/decision": "allow", "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"elasticsearch-operator.v5.8.19-66468cf8d7\" of ClusterRole \"elasticsearch-operator.v5.8.19-66468cf8d7\" to ServiceAccount \"elasticsearch-operator/openshift-operators-redhat\"" }, "apiVersion": "audit.k8s.io/v1", "auditID": "bbb765fb-93b8-436a-8e37-7de49748906b", "file": "/var/log/kube-apiserver/audit.log", "hostname": "master-2", "k8s_audit_level": "Metadata", "kind": "Event", "log_type": "audit", "objectRef": { "apiVersion": "v1", "name": "elasticsearch-metrics", "namespace": "openshift-logging", "resource": "services", "resourceVersion": "5436910", "uid": "4610bc36-1681-41f7-827f-5350390d4d04" }, "openshift": { "cluster_id": "657154e1-ef9c-426e-8079-d58eaee0a17b", "sequence": 63285 }, "requestReceivedTimestamp": "2025-04-01T17:52:36.672268Z", "requestURI": "/api/v1/namespaces/openshift-logging/services/elasticsearch-metrics", "responseStatus": { "code": 200, "metadata": {} }, "sourceIPs": [ "10.20.184.84" ], "source_type": "file", "stage": "ResponseComplete", "stageTimestamp": "2025-04-01T17:52:36.701486Z", "tag": ".k8s-audit.log", "user": { "extra": { "authentication.kubernetes.io/pod-name": [ "elasticsearch-operator-b78bc8575-2z9cz" ], "authentication.kubernetes.io/pod-uid": [ "d7de42e8-9cdd-4375-b923-f9450eb3f319" ] }, "groups": [ "system:serviceaccounts", "system:serviceaccounts:openshift-operators-redhat", "system:authenticated" ], "uid": "3ca20cad-7d98-4e03-a5e8-bde08b6751a4", "username": "system:serviceaccount:openshift-operators-redhat:elasticsearch-operator" }, "userAgent": "elasticsearch-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format", "verb": "update" }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-audit", "receiveTimestamp": "2025-04-01T17:52:37.723290209Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "master-2", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "timestamp": "2025-04-01T17:52:37.723290209Z" } ] ``` Infrastructure logs: ``` { "insertId": "1dg1frjf3bs28a", "jsonPayload": { "@timestamp": "2025-04-01T17:53:20.773567Z", "_RUNTIME_SCOPE": "system", "_STREAM_ID": "f90a5ee8d1034de79748f9f2741f6bef", "_SYSTEMD_INVOCATION_ID": "be81236e640547eea7533089fcfe9a7b", "hostname": "worker-0", "log_type": "infrastructure", "message": "W0401 17:53:20.773384 2844 sysfs.go:512] Cannot open /sys/bus/cpu/devices/cpu46/topology/physical_package_id, assuming 0 for physical_package_id of CPU 46", "openshift": { "cluster_id": "657154e1-ef9c-426e-8079-d58eaee0a17b", "sequence": 1291 }, "systemd": { "t": { "BOOT_ID": "7fe4f2f8cd16413888017b7a1e498b19", "CAP_EFFECTIVE": "1ffffffffff", "CMDLINE": "/usr/bin/kubelet --config=/etc/kubernetes/kubelet.conf --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig --container-runtime-endpoint=/var/run/crio/crio.sock --runtime-cgroups=/system.slice/crio.service --node-labels=node-role.kubernetes.io/worker,node.openshift.io/os_id=rhcos --node-ip=10.20.184.84 --minimum-container-ttl-duration=6m0s --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec --cloud-provider= --hostname-override= --provider-id= --pod-infra-container-image=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb2dea412db293ec52d0b2676bd57aeb105c619e47c7929ef008f6074bbd263e --system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=1Gi --v=2", "COMM": "kubelet", "EXE": "/usr/bin/kubelet", "GID": "0", "MACHINE_ID": "39f3d1eb565641e38343d63e53f9a348", "PID": "2844", "SELINUX_CONTEXT": "system_u:system_r:kubelet_t:s0-s0:c0.c1023", "STREAM_ID": "f90a5ee8d1034de79748f9f2741f6bef", "SYSTEMD_CGROUP": "/system.slice/kubelet.service", "SYSTEMD_INVOCATION_ID": "be81236e640547eea7533089fcfe9a7b", "SYSTEMD_SLICE": "system.slice", "SYSTEMD_UNIT": "kubelet.service", "TRANSPORT": "stdout", "UID": "0" }, "u": { "SYSLOG_FACILITY": "3", "SYSLOG_IDENTIFIER": "kubenswrapper" } }, "tag": ".journal.system", "time": "2025-04-01T17:53:20+00:00" }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-infra", "receiveTimestamp": "2025-04-01T17:53:22.167713879Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-04-01T17:53:22.167713879Z" }, { "insertId": "1dg1frjf3bs289", "jsonPayload": { "@timestamp": "2025-04-01T17:53:20.773567Z", "_RUNTIME_SCOPE": "system", "_STREAM_ID": "f90a5ee8d1034de79748f9f2741f6bef", "_SYSTEMD_INVOCATION_ID": "be81236e640547eea7533089fcfe9a7b", "hostname": "worker-0", "log_type": "infrastructure", "message": "W0401 17:53:20.773354 2844 sysfs.go:518] Cannot open /sys/bus/cpu/devices/cpu45/topology/physical_package_id, assuming 0 physical_package_id of CPU 45", "openshift": { "cluster_id": "657154e1-ef9c-426e-8079-d58eaee0a17b", "sequence": 1290 }, "systemd": { "t": { "BOOT_ID": "7fe4f2f8cd16413888017b7a1e498b19", "CAP_EFFECTIVE": "1ffffffffff", "CMDLINE": "/usr/bin/kubelet --config=/etc/kubernetes/kubelet.conf --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig --container-runtime-endpoint=/var/run/crio/crio.sock --runtime-cgroups=/system.slice/crio.service --node-labels=node-role.kubernetes.io/worker,node.openshift.io/os_id=rhcos --node-ip=10.20.184.84 --minimum-container-ttl-duration=6m0s --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec --cloud-provider= --hostname-override= --provider-id= --pod-infra-container-image=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb2dea412db293ec52d0b2676bd57aeb105c619e47c7929ef008f6074bbd263e --system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=1Gi --v=2", "COMM": "kubelet", "EXE": "/usr/bin/kubelet", "GID": "0", "MACHINE_ID": "39f3d1eb565641e38343d63e53f9a348", "PID": "2844", "SELINUX_CONTEXT": "system_u:system_r:kubelet_t:s0-s0:c0.c1023", "STREAM_ID": "f90a5ee8d1034de79748f9f2741f6bef", "SYSTEMD_CGROUP": "/system.slice/kubelet.service", "SYSTEMD_INVOCATION_ID": "be81236e640547eea7533089fcfe9a7b", "SYSTEMD_SLICE": "system.slice", "SYSTEMD_UNIT": "kubelet.service", "TRANSPORT": "stdout", "UID": "0" }, "u": { "SYSLOG_FACILITY": "3", "SYSLOG_IDENTIFIER": "kubenswrapper" } }, "tag": ".journal.system", "time": "2025-04-01T17:53:20+00:00" }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-infra", "receiveTimestamp": "2025-04-01T17:53:22.167713879Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-04-01T17:53:22.167713879Z" }, { "insertId": "1dg1frjf3bs288", "jsonPayload": { "@timestamp": "2025-04-01T17:53:20.773567Z", "_RUNTIME_SCOPE": "system", "_STREAM_ID": "f90a5ee8d1034de79748f9f2741f6bef", "_SYSTEMD_INVOCATION_ID": "be81236e640547eea7533089fcfe9a7b", "hostname": "worker-0", "log_type": "infrastructure", "message": "W0401 17:53:20.773333 2844 sysfs.go:512] Cannot open /sys/bus/cpu/devices/cpu45/topology/physical_package_id, assuming 0 for physical_package_id of CPU 45", "openshift": { "cluster_id": "657154e1-ef9c-426e-8079-d58eaee0a17b", "sequence": 1289 }, "systemd": { "t": { "BOOT_ID": "7fe4f2f8cd16413888017b7a1e498b19", "CAP_EFFECTIVE": "1ffffffffff", "CMDLINE": "/usr/bin/kubelet --config=/etc/kubernetes/kubelet.conf --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig --container-runtime-endpoint=/var/run/crio/crio.sock --runtime-cgroups=/system.slice/crio.service --node-labels=node-role.kubernetes.io/worker,node.openshift.io/os_id=rhcos --node-ip=10.20.184.84 --minimum-container-ttl-duration=6m0s --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec --cloud-provider= --hostname-override= --provider-id= --pod-infra-container-image=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eb2dea412db293ec52d0b2676bd57aeb105c619e47c7929ef008f6074bbd263e --system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=1Gi --v=2", "COMM": "kubelet", "EXE": "/usr/bin/kubelet", "GID": "0", "MACHINE_ID": "39f3d1eb565641e38343d63e53f9a348", "PID": "2844", "SELINUX_CONTEXT": "system_u:system_r:kubelet_t:s0-s0:c0.c1023", "STREAM_ID": "f90a5ee8d1034de79748f9f2741f6bef", "SYSTEMD_CGROUP": "/system.slice/kubelet.service", "SYSTEMD_INVOCATION_ID": "be81236e640547eea7533089fcfe9a7b", "SYSTEMD_SLICE": "system.slice", "SYSTEMD_UNIT": "kubelet.service", "TRANSPORT": "stdout", "UID": "0" }, "u": { "SYSLOG_FACILITY": "3", "SYSLOG_IDENTIFIER": "kubenswrapper" } }, "tag": ".journal.system", "time": "2025-04-01T17:53:20+00:00" }, "logName": "projects/hidden-bond-335716/logs/aosqe-logging-infra", "receiveTimestamp": "2025-04-01T17:53:22.167713879Z", "resource": { "labels": { "cluster_name": "", "location": "", "node_name": "worker-0", "project_id": "hidden-bond-335716" }, "type": "k8s_node" }, "severity": "INFO", "timestamp": "2025-04-01T17:53:22.167713879Z" } ```