Cluster logging 5.8.19 on OCP 4.14 with fluentd CSV and Pods: ``` [root@rdr-man-414log-bastion-0 ~]# oc get csv -n openshift-logging NAME DISPLAY VERSION REPLACES PHASE cluster-logging.v5.8.19 Red Hat OpenShift Logging 5.8.19 cluster-logging.v5.8.18 Succeeded cluster-observability-operator.v1.1.0 Cluster Observability Operator 1.1.0 cluster-observability-operator.v1.0.0 Succeeded elasticsearch-operator.v5.8.19 OpenShift Elasticsearch Operator 5.8.19 elasticsearch-operator.v5.8.18 Succeeded [root@rdr-man-414log-bastion-0 ~]# oc get pods -n openshift-logging NAME READY STATUS RESTARTS AGE cluster-logging-operator-9f4795575-km4bb 1/1 Running 0 9h collector-2vw26 1/1 Running 0 5m18s collector-l6wxm 1/1 Running 0 5m18s collector-qczdk 1/1 Running 0 5m18s collector-qv57h 1/1 Running 0 5m18s collector-wk2kw 1/1 Running 0 5m18s elasticsearch-cdm-t6btytco-1-5c78fb6974-hzv6h 2/2 Running 0 9h elasticsearch-cdm-t6btytco-2-7c466698bc-v8d5k 2/2 Running 0 9h elasticsearch-cdm-t6btytco-3-87c9f9956-8pzn9 2/2 Running 0 9h elasticsearch-im-app-29057370-dwlsg 0/1 Completed 0 11m elasticsearch-im-audit-29057370-xp8pj 0/1 Completed 0 11m elasticsearch-im-infra-29057370-gvm2f 0/1 Completed 0 11m kibana-866f44698d-6d2qq 2/2 Running 0 9h [root@rdr-man-414log-bastion-0 ~]# ``` Cluster Logging: CloudWatch: Application logs: ``` {'timestamp': 1743409904621, 'message': '{"@timestamp":"2025-03-31T08:31:35.205747642Z","group_name":"rdr-man-414log-zfvgp.application","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.209/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:d1\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.209/23\\",\\"gateway_ip\\":\\"10.131.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.209\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:d1\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://c6653404ef8cf5516c718c426a8ca87c54476b716bde1ff1498077d215d65dcd","container_image":"ibmcom/icp-mongodb-ppc64le:4.0.12","container_name":"acmeair-booking-db","labels":{"name":"acmeair-booking-db","pod-template-hash":"dbd46f4d4"},"namespace_id":"d6795e00-e70c-497f-8192-eff730aee32c","namespace_labels":{"kubernetes_io_metadata_name":"acme-air","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"acme-air","pod_id":"e451534c-60a5-4186-a00b-ff8a0c14b723","pod_ip":"10.131.0.209","pod_name":"acmeair-booking-db-dbd46f4d4-zpnzc","pod_owner":"ReplicaSet/acmeair-booking-db-dbd46f4d4"},"level":"default","log_type":"application","message":"2025-03-31T08:31:35.205+0000 I NETWORK [conn3] received client metadata from 10.131.0.208:36304 conn3: { driver: { name: \\"mongo-java-driver|legacy\\", version: \\"3.10.1\\" }, os: { type: \\"Linux\\", name: \\"Linux\\", architecture: \\"ppc64le\\", version: \\"5.14.0-284.109.1.el9_2.ppc64le\\" }, platform: \\"Java/Eclipse OpenJ9/1.8.0_262-b10\\" }","openshift":{"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-application"},"sequence":27905},"stream_name":"kubernetes.var.log.pods.acme-air_acmeair-booking-db-dbd46f4d4-zpnzc_e451534c-60a5-4186-a00b-ff8a0c14b723.acmeair-booking-db.0.log"}', 'ingestionTime': 1743409907228} {'timestamp': 1743409940473, 'message': '{"@timestamp":"2025-03-31T08:32:20.418812554Z","group_name":"rdr-man-414log-zfvgp.application","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.175/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:af\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.175/23\\",\\"gateway_ip\\":\\"10.131.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.175\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:af\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00","container_image":"quay.io/pravin_dsilva/centos:logtest","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"4566a766-d384-4fb9-8430-1c31bd2242b5","namespace_labels":{"kubernetes_io_metadata_name":"exclude-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"exclude-app","pod_id":"e522a963-f9e7-40e6-a502-fcaa5e58614f","pod_ip":"10.131.0.175","pod_name":"centos-logtest-gjd47","pod_owner":"ReplicationController/centos-logtest"},"level":"info","log_type":"application","message":"2025-03-31 08:32:20,418 - SVTLogger - INFO - centos-logtest-gjd47 : 248457 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu ","openshift":{"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-application"},"sequence":28056},"stream_name":"kubernetes.var.log.pods.exclude-app_centos-logtest-gjd47_e522a963-f9e7-40e6-a502-fcaa5e58614f.centos-logtest.0.log"}', 'ingestionTime': 1743409941709} {'timestamp': 1743409940735, 'message': '{"@timestamp":"2025-03-31T08:32:20.682337810Z","group_name":"rdr-man-414log-zfvgp.application","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.179/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:b3\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.179/23\\",\\"gateway_ip\\":\\"10.131.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.179\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:b3\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://f1ccdc3a273ba9353d7248786f079ea9e3ab6630f2a0ca75d20742ff1ba8cbdb","container_image":"quay.io/pravin_dsilva/centos:logtest","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5fa61797-a680-4391-b78c-6fddd25895dd","namespace_labels":{"kubernetes_io_metadata_name":"include-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"include-app","pod_id":"c9523cb3-4a8c-4582-9d2a-ade46a914d0f","pod_ip":"10.131.0.179","pod_name":"centos-logtest-2jxtm","pod_owner":"ReplicationController/centos-logtest"},"level":"info","log_type":"application","message":"2025-03-31 08:32:20,682 - SVTLogger - INFO - centos-logtest-2jxtm : 248444 : VgUkScyIZ zUCWCrDxy lfHgVb93A H9ujF2A0K 6kuVChDnI 0t18rVoGU jCLQBAShI dohYUbtkR Ni9lGhAlt gDrhOyELc QEjlCfT5w jtlbpta22 3fybZNOtK FUwwgooMw ram8Cu7LD RAizcGkaK wtBFsrIgq mECw2O1CK 6Y4xI1AxX QJx3tKj4y ","openshift":{"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-application"},"sequence":28057},"stream_name":"kubernetes.var.log.pods.include-app_centos-logtest-2jxtm_c9523cb3-4a8c-4582-9d2a-ade46a914d0f.centos-logtest.0.log"}', 'ingestionTime': 1743409941474} {'timestamp': 1743409941255, 'message': '{"@timestamp":"2025-03-31T08:32:21.071908878Z","group_name":"rdr-man-414log-zfvgp.application","hostname":"worker-1","kubernetes":{"annotations":{"k8s.ovn.org/pod-networks":"{\\"default\\":{\\"ip_addresses\\":[\\"10.131.0.174/23\\"],\\"mac_address\\":\\"0a:58:0a:83:00:ae\\",\\"gateway_ips\\":[\\"10.131.0.1\\"],\\"routes\\":[{\\"dest\\":\\"10.128.0.0/14\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"172.30.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"},{\\"dest\\":\\"100.64.0.0/16\\",\\"nextHop\\":\\"10.131.0.1\\"}],\\"ip_address\\":\\"10.131.0.174/23\\",\\"gateway_ip\\":\\"10.131.0.1\\"}}","k8s.v1.cni.cncf.io/network-status":"[{\\n \\"name\\": \\"ovn-kubernetes\\",\\n \\"interface\\": \\"eth0\\",\\n \\"ips\\": [\\n \\"10.131.0.174\\"\\n ],\\n \\"mac\\": \\"0a:58:0a:83:00:ae\\",\\n \\"default\\": true,\\n \\"dns\\": {}\\n}]","openshift.io/scc":"restricted-v2","seccomp.security.alpha.kubernetes.io/pod":"runtime/default"},"container_id":"cri-o://728df538617fa22d57cac9d28b79079398171cdd1f24897aec60fe0e43a8135e","container_image":"quay.io/pravin_dsilva/centos:logtest","container_name":"centos-logtest","labels":{"run":"centos-logtest","test":"centos-logtest"},"namespace_id":"5a1de092-661d-4860-b096-cf853ef15b38","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"namespace_name":"test-logging","pod_id":"5f049a1e-0586-4493-98a1-9329b8cf1d7b","pod_ip":"10.131.0.174","pod_name":"centos-logtest-76qb9","pod_owner":"ReplicationController/centos-logtest"},"level":"info","log_type":"application","message":"2025-03-31 08:32:21,071 - SVTLogger - INFO - centos-logtest-76qb9 : 248449 : vRbDfq1QU PJPUR4Hdp hH3OaPxCv bWmWEHEWa YjNzvWMhq nIaGsTX0O JdCXdYart RvxtSNW8m o19v45lVa yfjE1b5x3 or29hvNAM KpaFnINVU 6SWegLOIf SLSXVzt15 BNMU5DKjW VtclrkjmB hYbKkQHc1 GVWEqRSFL MM6y8mSzO d9wmAUElI ","openshift":{"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-application"},"sequence":28058},"stream_name":"kubernetes.var.log.pods.test-logging_centos-logtest-76qb9_5f049a1e-0586-4493-98a1-9329b8cf1d7b.centos-logtest.0.log"}', 'ingestionTime': 1743409941470} ``` Audit logs: ``` {'timestamp': 1743409945681, 'message': '{"@timestamp":"2025-03-31T08:32:21.370757630Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \\"system:openshift:operator:cloud-controller-manager\\" of ClusterRole \\"system:openshift:operator:cloud-controller-manager\\" to ServiceAccount \\"cluster-cloud-controller-manager/openshift-cloud-controller-manager-operator\\""},"apiVersion":"audit.k8s.io/v1","auditID":"d01af0a3-cef7-4341-882f-e822ae8dcedb","group_name":"rdr-man-414log-zfvgp.audit","hostname":"master-0","k8s_audit_level":"Metadata","kind":"Event","level":"Metadata","log_type":"audit","objectRef":{"apiGroup":"config.openshift.io","apiVersion":"v1","resource":"clusteroperators"},"openshift":{"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"},"sequence":87887},"requestReceivedTimestamp":"2025-03-31T08:32:21.357115Z","requestURI":"/apis/config.openshift.io/v1/clusteroperators?allowWatchBookmarks=true&resourceVersion=4661447&timeoutSeconds=340&watch=true","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.20.178.79"],"stage":"ResponseStarted","stageTimestamp":"2025-03-31T08:32:21.359110Z","stream_name":"master-0.k8s-audit.log","user":{"extra":{"authentication.kubernetes.io/pod-name":["cluster-cloud-controller-manager-operator-5488bb9dd5-z2mv7"],"authentication.kubernetes.io/pod-uid":["0ca7ee0e-0112-4ab7-8142-5196a82d06ad"]},"groups":["system:serviceaccounts","system:serviceaccounts:openshift-cloud-controller-manager-operator","system:authenticated"],"uid":"132db861-8e65-4e79-8437-dc1da4378f6f","username":"system:serviceaccount:openshift-cloud-controller-manager-operator:cluster-cloud-controller-manager"},"userAgent":"Go-http-client/2.0","verb":"watch"}', 'ingestionTime': 1743409946225} {'timestamp': 1743409872411, 'message': '{"@timestamp":"2025-03-31T08:30:55.878+00:00","audit.linux":{"record_id":"50275","type":"PROCTITLE"},"group_name":"rdr-man-414log-zfvgp.audit","hostname":"master-0","level":"default","log_type":"audit","message":"type=PROCTITLE msg=audit(1743409855.878:50275): proctitle=2F7573722F62696E2F72756E63002D2D73797374656D642D6367726F7570002D2D726F6F743D2F72756E2F72756E6300637265617465002D2D62756E646C65002F72756E2F636F6E7461696E6572732F73746F726167652F6F7665726C61792D636F6E7461696E6572732F396163656338313434353264346639663064353361","openshift":{"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"},"sequence":21501},"stream_name":"master-0.linux-audit.log"}', 'ingestionTime': 1743409873165} {'timestamp': 1743409950209, 'message': '{"@timestamp":"2025-03-31T08:32:20.692963110Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \\"basic-users\\" of ClusterRole \\"basic-user\\" to Group \\"system:authenticated\\""},"apiVersion":"audit.k8s.io/v1","auditID":"bd90afd7-2292-4255-8129-daaf18fbe733","group_name":"rdr-man-414log-zfvgp.audit","hostname":"master-0","kind":"Event","level":"Metadata","log_type":"audit","objectRef":{"apiGroup":"user.openshift.io","apiVersion":"v1","name":"~","resource":"users"},"openshift":{"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"},"sequence":94479},"openshift_audit_level":"Metadata","requestReceivedTimestamp":"2025-03-28T23:00:27.661973Z","requestURI":"/apis/user.openshift.io/v1/users/~","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.20.184.151","10.129.0.2"],"stage":"ResponseComplete","stageTimestamp":"2025-03-28T23:00:27.669155Z","stream_name":"master-0.openshift-audit.log","user":{"extra":{"authentication.kubernetes.io/pod-name":["prometheus-k8s-1"],"authentication.kubernetes.io/pod-uid":["7817ab63-f7f0-4031-8113-76972a7ba00d"]},"groups":["system:serviceaccounts","system:serviceaccounts:openshift-monitoring","system:authenticated"],"username":"system:serviceaccount:openshift-monitoring:prometheus-k8s"},"userAgent":"dockerregistry/v0.0.0 (linux/ppc64le) kubernetes/$Format","verb":"get"}', 'ingestionTime': 1743409951507} {'timestamp': 1743409950864, 'message': '{"@timestamp":"2025-03-31T08:32:11.425744513Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \\"system:openshift:operator:openshift-controller-manager-operator\\" of ClusterRole \\"cluster-admin\\" to ServiceAccount \\"openshift-controller-manager-operator/openshift-controller-manager-operator\\""},"apiVersion":"audit.k8s.io/v1","auditID":"bf41e513-adb6-4f6b-9ea7-0d451e29098d","group_name":"rdr-man-414log-zfvgp.audit","hostname":"master-1","k8s_audit_level":"Metadata","kind":"Event","level":"Metadata","log_type":"audit","objectRef":{"apiVersion":"v1","name":"route-controller-manager","namespace":"openshift-route-controller-manager","resource":"services"},"openshift":{"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"},"sequence":129576},"requestReceivedTimestamp":"2025-03-31T06:34:40.168253Z","requestURI":"/api/v1/namespaces/openshift-route-controller-manager/services/route-controller-manager","responseStatus":{"code":200,"metadata":{}},"sourceIPs":["10.130.1.229"],"stage":"ResponseComplete","stageTimestamp":"2025-03-31T06:34:40.171370Z","stream_name":"master-1.k8s-audit.log","user":{"extra":{"authentication.kubernetes.io/pod-name":["openshift-controller-manager-operator-559df8b94f-v4rw6"],"authentication.kubernetes.io/pod-uid":["97fd249a-23ac-44e7-a12b-3b4f01d9c50f"]},"groups":["system:serviceaccounts","system:serviceaccounts:openshift-controller-manager-operator","system:authenticated"],"uid":"b7bc7fe2-377e-4b06-869b-0f562868fd4d","username":"system:serviceaccount:openshift-controller-manager-operator:openshift-controller-manager-operator"},"userAgent":"cluster-openshift-controller-manager-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","verb":"get"}', 'ingestionTime': 1743409951423} ``` Infrastructure logs: ``` {'timestamp': 1743409933752, 'message': '{"@timestamp":"2025-03-31T08:32:11.219887Z","SYSLOG_TIMESTAMP":"Mar 31 08:32:11 ","_RUNTIME_SCOPE":"system","_SYSTEMD_INVOCATION_ID":"bc1640f3ccf741d08f646d1fc308826e","group_name":"rdr-man-414log-zfvgp.infrastructure","hostname":"master-0","level":"notice","log_type":"infrastructure","message":"ovs|74905|connmgr|INFO|br-ex<->unix#382801: 2 flow_mods in the last 0 s (2 adds)","openshift":{"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-infrastructure"},"sequence":77033},"stream_name":"master-0.journal.system","systemd":{"t":{"BOOT_ID":"8d72e764ba3e48df82758744e34410b1","CAP_EFFECTIVE":"7c00","CMDLINE":"ovs-vswitchd unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --user openvswitch:hugetlbfs --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach","COMM":"ovs-vswitchd","EXE":"/usr/sbin/ovs-vswitchd","GID":"801","MACHINE_ID":"4892829894aa40568697ad73dd5d7784","PID":"1108","SELINUX_CONTEXT":"system_u:system_r:openvswitch_t:s0","SYSTEMD_CGROUP":"/system.slice/ovs-vswitchd.service","SYSTEMD_INVOCATION_ID":"bc1640f3ccf741d08f646d1fc308826e","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"ovs-vswitchd.service","TRANSPORT":"syslog","UID":"800"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"ovs-vswitchd"}},"time":"2025-03-31T08:32:11+00:00"}', 'ingestionTime': 1743409936393} {'timestamp': 1743409942850, 'message': '{"@timestamp":"2025-03-31T08:32:20.812857331Z","group_name":"rdr-man-414log-zfvgp.infrastructure","hostname":"master-0","kubernetes":{"annotations":{"openshift.io/scc":"hostaccess"},"container_id":"cri-o://410531f7bc64f59c7fdc5fa4b6cdb828336accf1ec5d71c387a9c9766bce6f84","container_image":"quay.io/openshift-release-dev/ocp-release@sha256:b05ebaa64e24c93704a8879e0fc9561029433aa1face919c5a1283bac316723c","container_name":"cluster-version-operator","labels":{"k8s-app":"cluster-version-operator","pod-template-hash":"86dc4548b4"},"namespace_id":"4b1cfb67-16ee-48d4-8ea0-8397a3bb995e","namespace_labels":{"kubernetes_io_metadata_name":"openshift-cluster-version","name":"openshift-cluster-version","olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-cluster-version","pod_id":"4ab7feb2-60f1-4143-ae52-e77a40d9e8e9","pod_ip":"10.20.184.44","pod_name":"cluster-version-operator-86dc4548b4-fqtzx","pod_owner":"ReplicaSet/cluster-version-operator-86dc4548b4"},"level":"info","log_type":"infrastructure","message":"I0331 08:32:20.812561 1 cvo.go:625] Finished syncing cluster version \\"openshift-cluster-version/version\\" (822.416µs)","openshift":{"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-infrastructure"},"sequence":84449},"stream_name":"master-0.kubernetes.var.log.pods.openshift-cluster-version_cluster-version-operator-86dc4548b4-fqtzx_4ab7feb2-60f1-4143-ae52-e77a40d9e8e9.cluster-version-operator.0.log"}', 'ingestionTime': 1743409944691} {'timestamp': 1743409876747, 'message': '{"@timestamp":"2025-03-31T08:30:54.692488538Z","group_name":"rdr-man-414log-zfvgp.infrastructure","hostname":"master-0","kubernetes":{"annotations":{"kubectl.kubernetes.io/default-container":"etcd","kubernetes.io/config.hash":"138f360b01ed4ba4bfb314ef847e6c70","kubernetes.io/config.mirror":"138f360b01ed4ba4bfb314ef847e6c70","kubernetes.io/config.seen":"2025-03-21T06:57:41.182451900Z","kubernetes.io/config.source":"file","target.workload.openshift.io/management":"{\\"effect\\": \\"PreferredDuringScheduling\\"}"},"container_id":"cri-o://37c5b8a1cb3af5a9283a9ab869383fe5247e99b0d84826a3ecd2c8cdedeb543f","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e2023d838c031de2cfd3505ef82767f5ec6a58bbfbf03a076d41a383126279ab","container_name":"etcd","labels":{"app":"etcd","etcd":"true","k8s-app":"etcd","revision":"9"},"namespace_id":"d038e3e1-6f32-4a77-8566-42325fbc2a3d","namespace_labels":{"kubernetes_io_metadata_name":"openshift-etcd","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-etcd","pod_id":"9ebd9aa4-2ffb-4d56-a5a2-22b22960f659","pod_ip":"10.20.184.44","pod_name":"etcd-master-0","pod_owner":"Node/master-0"},"level":"info","log_type":"infrastructure","message":"{\\"level\\":\\"info\\",\\"ts\\":\\"2025-03-31T08:30:54.692276Z\\",\\"caller\\":\\"fileutil/purge.go:96\\",\\"msg\\":\\"purged\\",\\"path\\":\\"/var/lib/etcd/member/snap/0000000000000007-0000000000532c34.snap\\"}","openshift":{"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-infrastructure"},"sequence":32371},"stream_name":"master-0.kubernetes.var.log.pods.openshift-etcd_etcd-master-0_138f360b01ed4ba4bfb314ef847e6c70.etcd.0.log"}', 'ingestionTime': 1743409883731} {'timestamp': 1743409898403, 'message': '{"@timestamp":"2025-03-31T08:31:32.725159082Z","group_name":"rdr-man-414log-zfvgp.infrastructure","hostname":"master-0","kubernetes":{"annotations":{"openshift.io/scc":"privileged"},"container_id":"cri-o://d6705fc652f86e4004bdb369f8cc04b28d4b202e678e458c5ffed43b88556208","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3b27759261d7bc22d12537a8ce89e3edf16bd89875766fde58228a05b0b347b","container_name":"node-ca","labels":{"controller-revision-hash":"5c8fccbfd4","name":"node-ca","pod-template-generation":"1"},"namespace_id":"f2134aa0-7fba-4da3-86fa-444c5e49c413","namespace_labels":{"kubernetes_io_metadata_name":"openshift-image-registry","olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"namespace_name":"openshift-image-registry","pod_id":"e38651c7-0965-4cd0-b591-4d7f46b17911","pod_ip":"10.20.184.44","pod_name":"node-ca-bzhf8","pod_owner":"DaemonSet/node-ca"},"level":"default","log_type":"infrastructure","message":"image-registry.openshift-image-registry.svc:5000","openshift":{"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-infrastructure"},"sequence":46184},"stream_name":"master-0.kubernetes.var.log.pods.openshift-image-registry_node-ca-bzhf8_e38651c7-0965-4cd0-b591-4d7f46b17911.node-ca.0.log"}', 'ingestionTime': 1743409901661} ``` Fluentd: Application logs: ``` 2025-04-01T07:32:02+00:00 kubernetes.var.log.pods.include-app_centos-logtest-2jxtm_c9523cb3-4a8c-4582-9d2a-ade46a914d0f.centos-logtest.0.log {"@timestamp":"2025-04-01T07:32:02.765526776+00:00","message":"2025-04-01 07:32:02,765 - SVTLogger - INFO - centos-logtest-2jxtm : 331099 : VgUkScyIZ zUCWCrDxy lfHgVb93A H9ujF2A0K 6kuVChDnI 0t18rVoGU jCLQBAShI dohYUbtkR Ni9lGhAlt gDrhOyELc QEjlCfT5w jtlbpta22 3fybZNOtK FUwwgooMw ram8Cu7LD RAizcGkaK wtBFsrIgq mECw2O1CK 6Y4xI1AxX QJx3tKj4y ","docker":{"container_id":"f1ccdc3a273ba9353d7248786f079ea9e3ab6630f2a0ca75d20742ff1ba8cbdb"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"include-app","pod_name":"centos-logtest-2jxtm","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"c9523cb3-4a8c-4582-9d2a-ade46a914d0f","pod_ip":"10.131.0.179","host":"worker-1","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"5fa61797-a680-4391-b78c-6fddd25895dd","namespace_labels":{"kubernetes_io_metadata_name":"include-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T07:32:02.766274+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":5631,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"clusterId":"ocp47-log507-19","logs":"5.8_414-app"}},"viaq_msg_id":"MjYwYWQ2Y2QtNDRhNS00MzJhLTk3MzctYzYxMGJhOGUyMjky","log_type":"application"} 2025-04-01T07:32:03+00:00 kubernetes.var.log.pods.include-app_centos-logtest-2jxtm_c9523cb3-4a8c-4582-9d2a-ade46a914d0f.centos-logtest.0.log {"@timestamp":"2025-04-01T07:32:03.766949238+00:00","message":"2025-04-01 07:32:03,766 - SVTLogger - INFO - centos-logtest-2jxtm : 331100 : VgUkScyIZ zUCWCrDxy lfHgVb93A H9ujF2A0K 6kuVChDnI 0t18rVoGU jCLQBAShI dohYUbtkR Ni9lGhAlt gDrhOyELc QEjlCfT5w jtlbpta22 3fybZNOtK FUwwgooMw ram8Cu7LD RAizcGkaK wtBFsrIgq mECw2O1CK 6Y4xI1AxX QJx3tKj4y ","docker":{"container_id":"f1ccdc3a273ba9353d7248786f079ea9e3ab6630f2a0ca75d20742ff1ba8cbdb"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"include-app","pod_name":"centos-logtest-2jxtm","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"c9523cb3-4a8c-4582-9d2a-ade46a914d0f","pod_ip":"10.131.0.179","host":"worker-1","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"5fa61797-a680-4391-b78c-6fddd25895dd","namespace_labels":{"kubernetes_io_metadata_name":"include-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T07:32:03.767519+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":5634,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"clusterId":"ocp47-log507-19","logs":"5.8_414-app"}},"viaq_msg_id":"M2Y4OWFiOWUtOTUyNC00NWE2LWE4NTctNmZiZDNjZmVkOWUy","log_type":"application"} 2025-04-01T07:32:04+00:00 kubernetes.var.log.pods.include-app_centos-logtest-2jxtm_c9523cb3-4a8c-4582-9d2a-ade46a914d0f.centos-logtest.0.log {"@timestamp":"2025-04-01T07:32:04.768559226+00:00","message":"2025-04-01 07:32:04,768 - SVTLogger - INFO - centos-logtest-2jxtm : 331101 : VgUkScyIZ zUCWCrDxy lfHgVb93A H9ujF2A0K 6kuVChDnI 0t18rVoGU jCLQBAShI dohYUbtkR Ni9lGhAlt gDrhOyELc QEjlCfT5w jtlbpta22 3fybZNOtK FUwwgooMw ram8Cu7LD RAizcGkaK wtBFsrIgq mECw2O1CK 6Y4xI1AxX QJx3tKj4y ","docker":{"container_id":"f1ccdc3a273ba9353d7248786f079ea9e3ab6630f2a0ca75d20742ff1ba8cbdb"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"include-app","pod_name":"centos-logtest-2jxtm","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"c9523cb3-4a8c-4582-9d2a-ade46a914d0f","pod_ip":"10.131.0.179","host":"worker-1","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"5fa61797-a680-4391-b78c-6fddd25895dd","namespace_labels":{"kubernetes_io_metadata_name":"include-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T07:32:04.769484+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":5639,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"clusterId":"ocp47-log507-19","logs":"5.8_414-app"}},"viaq_msg_id":"NjlhMWJmNTItMjNhMy00MjIzLTgzZmQtZTMyYzVlN2Y3NzVk","log_type":"application"} 2025-04-01T07:32:05+00:00 kubernetes.var.log.pods.include-app_centos-logtest-2jxtm_c9523cb3-4a8c-4582-9d2a-ade46a914d0f.centos-logtest.0.log {"@timestamp":"2025-04-01T07:32:05.769284992+00:00","message":"2025-04-01 07:32:05,768 - SVTLogger - INFO - centos-logtest-2jxtm : 331102 : VgUkScyIZ zUCWCrDxy lfHgVb93A H9ujF2A0K 6kuVChDnI 0t18rVoGU jCLQBAShI dohYUbtkR Ni9lGhAlt gDrhOyELc QEjlCfT5w jtlbpta22 3fybZNOtK FUwwgooMw ram8Cu7LD RAizcGkaK wtBFsrIgq mECw2O1CK 6Y4xI1AxX QJx3tKj4y ","docker":{"container_id":"f1ccdc3a273ba9353d7248786f079ea9e3ab6630f2a0ca75d20742ff1ba8cbdb"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"include-app","pod_name":"centos-logtest-2jxtm","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"c9523cb3-4a8c-4582-9d2a-ade46a914d0f","pod_ip":"10.131.0.179","host":"worker-1","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"5fa61797-a680-4391-b78c-6fddd25895dd","namespace_labels":{"kubernetes_io_metadata_name":"include-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T07:32:05.769874+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":5642,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"clusterId":"ocp47-log507-19","logs":"5.8_414-app"}},"viaq_msg_id":"NGU4MWZkZDctMDhiNy00NTQ1LThkNjAtMjE0YTU2MmEwYjc5","log_type":"application"} ``` Audit logs: ``` 2025-04-01T07:38:31+00:00 k8s-audit.log {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"b5e82c02-57a3-4f01-ae05-a51d41a299f7","stage":"ResponseComplete","requestURI":"/apis/logging.openshift.io/v1/namespaces/openshift-logging/elasticsearches/elasticsearch/status","verb":"update","user":{"username":"system:serviceaccount:openshift-operators-redhat:elasticsearch-operator","uid":"3ca20cad-7d98-4e03-a5e8-bde08b6751a4","groups":["system:serviceaccounts","system:serviceaccounts:openshift-operators-redhat","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["elasticsearch-operator-b78bc8575-2z9cz"],"authentication.kubernetes.io/pod-uid":["d7de42e8-9cdd-4375-b923-f9450eb3f319"]}},"sourceIPs":["10.20.184.84"],"userAgent":"elasticsearch-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"elasticsearches","namespace":"openshift-logging","name":"elasticsearch","uid":"4533f8b2-970d-41d0-93c7-8304e387d87f","apiGroup":"logging.openshift.io","apiVersion":"v1","resourceVersion":"5212934","subresource":"status"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-04-01T07:38:31.131404Z","stageTimestamp":"2025-04-01T07:38:31.144137Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"elasticsearch-operator.v5.8.19-66468cf8d7\" of ClusterRole \"elasticsearch-operator.v5.8.19-66468cf8d7\" to ServiceAccount \"elasticsearch-operator/openshift-operators-redhat\""},"@timestamp":"2025-04-01T07:38:31.131404Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-2","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.131","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T07:38:31.145022+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":3911,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"clusterId":"ocp47-log507-19","logs":"5.8_414-audit"}},"viaq_msg_id":"MTIwZDlhZTYtMjQ1Yi00NGQ0LWFhNmMtYTM5MjViYjhmYWMz","log_type":"audit"} 2025-04-01T07:38:31+00:00 k8s-audit.log {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"e641ccee-9e86-4a3f-9633-209167796b69","stage":"ResponseComplete","requestURI":"/apis/apps/v1/namespaces/openshift-logging/deployments","verb":"create","user":{"username":"system:serviceaccount:openshift-operators-redhat:elasticsearch-operator","uid":"3ca20cad-7d98-4e03-a5e8-bde08b6751a4","groups":["system:serviceaccounts","system:serviceaccounts:openshift-operators-redhat","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["elasticsearch-operator-b78bc8575-2z9cz"],"authentication.kubernetes.io/pod-uid":["d7de42e8-9cdd-4375-b923-f9450eb3f319"]}},"sourceIPs":["10.20.184.84"],"userAgent":"elasticsearch-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"deployments","namespace":"openshift-logging","name":"elasticsearch-cdm-t6btytco-1","apiGroup":"apps","apiVersion":"v1"},"responseStatus":{"metadata":{},"status":"Failure","message":"deployments.apps \"elasticsearch-cdm-t6btytco-1\" already exists","reason":"AlreadyExists","details":{"name":"elasticsearch-cdm-t6btytco-1","group":"apps","kind":"deployments"},"code":409},"requestReceivedTimestamp":"2025-04-01T07:38:31.145753Z","stageTimestamp":"2025-04-01T07:38:31.155760Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"elasticsearch-operator.v5.8.19-66468cf8d7\" of ClusterRole \"elasticsearch-operator.v5.8.19-66468cf8d7\" to ServiceAccount \"elasticsearch-operator/openshift-operators-redhat\""},"@timestamp":"2025-04-01T07:38:31.145753Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-2","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.131","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T07:38:31.156245+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":3912,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"clusterId":"ocp47-log507-19","logs":"5.8_414-audit"}},"viaq_msg_id":"YWI5OWJiODgtY2FhNC00MjAzLThkMjAtMWM5ZDczNTg3N2U0","log_type":"audit"} 2025-04-01T07:38:31+00:00 k8s-audit.log {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"36b296b0-740b-4827-b3cd-fae199698ec5","stage":"ResponseComplete","requestURI":"/apis/apps/v1/namespaces/openshift-logging/deployments","verb":"create","user":{"username":"system:serviceaccount:openshift-operators-redhat:elasticsearch-operator","uid":"3ca20cad-7d98-4e03-a5e8-bde08b6751a4","groups":["system:serviceaccounts","system:serviceaccounts:openshift-operators-redhat","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["elasticsearch-operator-b78bc8575-2z9cz"],"authentication.kubernetes.io/pod-uid":["d7de42e8-9cdd-4375-b923-f9450eb3f319"]}},"sourceIPs":["10.20.184.84"],"userAgent":"elasticsearch-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"deployments","namespace":"openshift-logging","name":"elasticsearch-cdm-t6btytco-2","apiGroup":"apps","apiVersion":"v1"},"responseStatus":{"metadata":{},"status":"Failure","message":"deployments.apps \"elasticsearch-cdm-t6btytco-2\" already exists","reason":"AlreadyExists","details":{"name":"elasticsearch-cdm-t6btytco-2","group":"apps","kind":"deployments"},"code":409},"requestReceivedTimestamp":"2025-04-01T07:38:31.157076Z","stageTimestamp":"2025-04-01T07:38:31.165782Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"elasticsearch-operator.v5.8.19-66468cf8d7\" of ClusterRole \"elasticsearch-operator.v5.8.19-66468cf8d7\" to ServiceAccount \"elasticsearch-operator/openshift-operators-redhat\""},"@timestamp":"2025-04-01T07:38:31.157076Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-2","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.131","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T07:38:31.166316+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":3913,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"clusterId":"ocp47-log507-19","logs":"5.8_414-audit"}},"viaq_msg_id":"ZmQ1MzNjYTMtOGIzZi00NzhmLWE5ZGYtNTcyOTY3MDc4MjQx","log_type":"audit"} 2025-04-01T07:38:27+00:00 openshift-audit.log {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"526ff681-a06d-4b43-bc94-7085a93dc3ef","stage":"ResponseComplete","requestURI":"/apis/user.openshift.io/v1/users/~","verb":"get","user":{"username":"system:serviceaccount:openshift-monitoring:prometheus-k8s","groups":["system:serviceaccounts","system:serviceaccounts:openshift-monitoring","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["prometheus-k8s-1"],"authentication.kubernetes.io/pod-uid":["7817ab63-f7f0-4031-8113-76972a7ba00d"]}},"sourceIPs":["10.20.184.151","10.129.0.2"],"userAgent":"dockerregistry/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"users","name":"~","apiGroup":"user.openshift.io","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-04-01T07:38:27.653499Z","stageTimestamp":"2025-04-01T07:38:27.663694Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"basic-users\" of ClusterRole \"basic-user\" to Group \"system:authenticated\""},"@timestamp":"2025-04-01T07:38:27.653499Z","openshift_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T07:38:27.665202+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":31366,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"clusterId":"ocp47-log507-19","logs":"5.8_414-audit"}},"viaq_msg_id":"YjNmZTI1ZTktZDk3MS00MDI1LWJkZGEtODEwNmUwMWU5MmU1","log_type":"audit"} ``` Infrastructure logs: ``` 2025-04-01T07:51:28+00:00 kubernetes.var.log.pods.openshift-monitoring_node-exporter-s4wp6_e0096717-2b78-4c0f-bd13-63b7cf6767f5.node-exporter.0.log {"@timestamp":"2025-04-01T07:51:28.316059345+00:00","message":"ts=2025-04-01T07:51:28.315Z caller=collector.go:169 level=error msg=\"collector failed\" name=fibrechannel duration_seconds=0.000803667 err=\"error obtaining FibreChannel class info: failed to read file \\\"/host/sys/class/fc_host/host1/symbolic_name\\\": open /host/sys/class/fc_host/host1/symbolic_name: no such file or directory\"","docker":{"container_id":"d8b51a3ad5a5332de6a682e7f6bff12c892dae5f151fd301c42591c5c2a2d15e"},"kubernetes":{"container_name":"node-exporter","namespace_name":"openshift-monitoring","pod_name":"node-exporter-s4wp6","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9816ed2e1057d2c0e1967bb7806baa2f46bb8f2772b5e9d2b9c5d20c31d93e5e","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9816ed2e1057d2c0e1967bb7806baa2f46bb8f2772b5e9d2b9c5d20c31d93e5e","pod_id":"e0096717-2b78-4c0f-bd13-63b7cf6767f5","pod_ip":"10.20.184.44","host":"master-0","labels":{"app_kubernetes_io_component":"exporter","app_kubernetes_io_managed-by":"cluster-monitoring-operator","app_kubernetes_io_name":"node-exporter","app_kubernetes_io_part-of":"openshift-monitoring","app_kubernetes_io_version":"1.6.1","controller-revision-hash":"84544887c4","pod-template-generation":"1"},"master_url":"https://kubernetes.default.svc","namespace_id":"ac86bef9-a349-4fb7-bf2b-24c34b3c3d3c","namespace_labels":{"kubernetes_io_metadata_name":"openshift-monitoring","name":"openshift-monitoring","network_openshift_io_policy-group":"monitoring","olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["app_kubernetes_io_component=exporter","app_kubernetes_io_managed-by=cluster-monitoring-operator","app_kubernetes_io_name=node-exporter","app_kubernetes_io_part-of=openshift-monitoring","app_kubernetes_io_version=1_6_1","controller-revision-hash=84544887c4","pod-template-generation=1"]},"level":"error","hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.44","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T07:51:28.318872+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":16913,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"clusterId":"ocp47-log507-19","logs":"5.8_414-infra"}},"viaq_msg_id":"MzhmNDJmNzctMDgyYy00ZmYzLWJjZTQtNTBlZGFlZTc3NTEz","log_type":"infrastructure"} 2025-04-01T07:51:25+00:00 kubernetes.var.log.pods.openshift-image-registry_node-ca-cdbn2_485d7884-ad2d-4adc-a23a-0845864fd46a.node-ca.0.log {"@timestamp":"2025-04-01T07:51:25.300520535+00:00","message":"image-registry.openshift-image-registry.svc..5000","docker":{"container_id":"a8beb3e869033f53f48a9862ff402c5d9bb3bfe9c9f6422a4f6657334d2f8a2a"},"kubernetes":{"container_name":"node-ca","namespace_name":"openshift-image-registry","pod_name":"node-ca-cdbn2","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3b27759261d7bc22d12537a8ce89e3edf16bd89875766fde58228a05b0b347b","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3b27759261d7bc22d12537a8ce89e3edf16bd89875766fde58228a05b0b347b","pod_id":"485d7884-ad2d-4adc-a23a-0845864fd46a","pod_ip":"10.20.184.131","host":"master-2","labels":{"controller-revision-hash":"5c8fccbfd4","name":"node-ca","pod-template-generation":"1"},"master_url":"https://kubernetes.default.svc","namespace_id":"f2134aa0-7fba-4da3-86fa-444c5e49c413","namespace_labels":{"kubernetes_io_metadata_name":"openshift-image-registry","olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["controller-revision-hash=5c8fccbfd4","name=node-ca","pod-template-generation=1"]},"level":"unknown","hostname":"master-2","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.131","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T07:51:25.301549+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1043,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"clusterId":"ocp47-log507-19","logs":"5.8_414-infra"}},"viaq_msg_id":"ZmZkMDFmMzMtZjg0Yi00MjRkLTgzMTctYTZhMDBhYjc3ZmNh","log_type":"infrastructure"} 2025-04-01T07:51:25+00:00 kubernetes.var.log.pods.openshift-image-registry_node-ca-cdbn2_485d7884-ad2d-4adc-a23a-0845864fd46a.node-ca.0.log {"@timestamp":"2025-04-01T07:51:25.304753500+00:00","message":"image-registry.openshift-image-registry.svc.cluster.local..5000","docker":{"container_id":"a8beb3e869033f53f48a9862ff402c5d9bb3bfe9c9f6422a4f6657334d2f8a2a"},"kubernetes":{"container_name":"node-ca","namespace_name":"openshift-image-registry","pod_name":"node-ca-cdbn2","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3b27759261d7bc22d12537a8ce89e3edf16bd89875766fde58228a05b0b347b","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3b27759261d7bc22d12537a8ce89e3edf16bd89875766fde58228a05b0b347b","pod_id":"485d7884-ad2d-4adc-a23a-0845864fd46a","pod_ip":"10.20.184.131","host":"master-2","labels":{"controller-revision-hash":"5c8fccbfd4","name":"node-ca","pod-template-generation":"1"},"master_url":"https://kubernetes.default.svc","namespace_id":"f2134aa0-7fba-4da3-86fa-444c5e49c413","namespace_labels":{"kubernetes_io_metadata_name":"openshift-image-registry","olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["controller-revision-hash=5c8fccbfd4","name=node-ca","pod-template-generation=1"]},"level":"unknown","hostname":"master-2","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.131","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T07:51:25.305574+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1044,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"clusterId":"ocp47-log507-19","logs":"5.8_414-infra"}},"viaq_msg_id":"YjE0N2M3YjItZTE4OC00NmJkLTkyYjQtMDE5NmI4YmJjM2Zj","log_type":"infrastructure"} 2025-04-01T07:51:25+00:00 kubernetes.var.log.pods.openshift-image-registry_node-ca-cdbn2_485d7884-ad2d-4adc-a23a-0845864fd46a.node-ca.0.log {"@timestamp":"2025-04-01T07:51:25.311763312+00:00","message":"image-registry.openshift-image-registry.svc.cluster.local:5000","docker":{"container_id":"a8beb3e869033f53f48a9862ff402c5d9bb3bfe9c9f6422a4f6657334d2f8a2a"},"kubernetes":{"container_name":"node-ca","namespace_name":"openshift-image-registry","pod_name":"node-ca-cdbn2","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3b27759261d7bc22d12537a8ce89e3edf16bd89875766fde58228a05b0b347b","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3b27759261d7bc22d12537a8ce89e3edf16bd89875766fde58228a05b0b347b","pod_id":"485d7884-ad2d-4adc-a23a-0845864fd46a","pod_ip":"10.20.184.131","host":"master-2","labels":{"controller-revision-hash":"5c8fccbfd4","name":"node-ca","pod-template-generation":"1"},"master_url":"https://kubernetes.default.svc","namespace_id":"f2134aa0-7fba-4da3-86fa-444c5e49c413","namespace_labels":{"kubernetes_io_metadata_name":"openshift-image-registry","olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["controller-revision-hash=5c8fccbfd4","name=node-ca","pod-template-generation=1"]},"level":"unknown","hostname":"master-2","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.131","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T07:51:25.312549+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1045,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"clusterId":"ocp47-log507-19","logs":"5.8_414-infra"}},"viaq_msg_id":"MzgwY2QwN2YtN2Y1Zi00ZGQyLThkYWUtNjMzYzYxOWM3Y2Yw","log_type":"infrastructure"} 2025-04-01T07:51:25+00:00 kubernetes.var.log.pods.openshift-image-registry_node-ca-cdbn2_485d7884-ad2d-4adc-a23a-0845864fd46a.node-ca.0.log {"@timestamp":"2025-04-01T07:51:25.314278714+00:00","message":"image-registry.openshift-image-registry.svc:5000","docker":{"container_id":"a8beb3e869033f53f48a9862ff402c5d9bb3bfe9c9f6422a4f6657334d2f8a2a"},"kubernetes":{"container_name":"node-ca","namespace_name":"openshift-image-registry","pod_name":"node-ca-cdbn2","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3b27759261d7bc22d12537a8ce89e3edf16bd89875766fde58228a05b0b347b","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3b27759261d7bc22d12537a8ce89e3edf16bd89875766fde58228a05b0b347b","pod_id":"485d7884-ad2d-4adc-a23a-0845864fd46a","pod_ip":"10.20.184.131","host":"master-2","labels":{"controller-revision-hash":"5c8fccbfd4","name":"node-ca","pod-template-generation":"1"},"master_url":"https://kubernetes.default.svc","namespace_id":"f2134aa0-7fba-4da3-86fa-444c5e49c413","namespace_labels":{"kubernetes_io_metadata_name":"openshift-image-registry","olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["controller-revision-hash=5c8fccbfd4","name=node-ca","pod-template-generation=1"]},"level":"unknown","hostname":"master-2","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.131","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T07:51:25.314710+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1046,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"clusterId":"ocp47-log507-19","logs":"5.8_414-infra"}},"viaq_msg_id":"MDk3M2YxYWQtNDJlYi00ZDFmLTg4ZWItMjJhMGM4OWExZDlh","log_type":"infrastructure"} ``` Syslog: Application logs: ``` Apr 1 08:09:08 worker-1 ocp47.22[myproc] {"@timestamp":"2025-04-01T08:09:07.306000342+00:00","message":"2025-04-01 08:09:07,305 - SVTLogger - INFO - centos-logtest-2jxtm : 333320 : VgUkScyIZ zUCWCrDxy lfHgVb93A H9ujF2A0K 6kuVChDnI 0t18rVoGU jCLQBAShI dohYUbtkR Ni9lGhAlt gDrhOyELc QEjlCfT5w jtlbpta22 3fybZNOtK FUwwgooMw ram8Cu7LD RAizcGkaK wtBFsrIgq mECw2O1CK 6Y4xI1AxX QJx3tKj4y ","docker":{"container_id":"f1ccdc3a273ba9353d7248786f079ea9e3ab6630f2a0ca75d20742ff1ba8cbdb"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"include-app","pod_name":"centos-logtest-2jxtm","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"c9523cb3-4a8c-4582-9d2a-ade46a914d0f","pod_ip":"10.131.0.179","host":"worker-1","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"5fa61797-a680-4391-b78c-6fddd25895dd","namespace_labels":{"kubernetes_io_metadata_name":"include-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:09:07.307478+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":669,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_app","secure":"true","syslog":"east"}},"viaq_msg_id":"NGYwYWU4OGMtYTI2Yy00YzVmLTkzNjQtZWJhMTFiZjk1ZmJh","log_type":"application"} Apr 1 08:09:08 worker-1 ocp47.22[myproc] {"@timestamp":"2025-04-01T08:09:08.003187274+00:00","message":"2025-04-01 08:09:08,002 - SVTLogger - INFO - centos-logtest-gjd47 : 333340 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu ","docker":{"container_id":"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"exclude-app","pod_name":"centos-logtest-gjd47","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"e522a963-f9e7-40e6-a502-fcaa5e58614f","pod_ip":"10.131.0.175","host":"worker-1","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"4566a766-d384-4fb9-8430-1c31bd2242b5","namespace_labels":{"kubernetes_io_metadata_name":"exclude-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:09:08.005775+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":670,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_app","secure":"true","syslog":"east"}},"viaq_msg_id":"NzZiYmQzZGUtY2FiYS00OTk5LWExZTEtZjMwN2M2ZWM0NzMy","log_type":"application"} Apr 1 08:09:08 worker-1 ocp47.22[myproc] {"@timestamp":"2025-04-01T08:09:08.003187182+00:00","message":"2025-04-01 08:09:08,002 - SVTLogger - INFO - centos-logtest-76qb9 : 333330 : vRbDfq1QU PJPUR4Hdp hH3OaPxCv bWmWEHEWa YjNzvWMhq nIaGsTX0O JdCXdYart RvxtSNW8m o19v45lVa yfjE1b5x3 or29hvNAM KpaFnINVU 6SWegLOIf SLSXVzt15 BNMU5DKjW VtclrkjmB hYbKkQHc1 GVWEqRSFL MM6y8mSzO d9wmAUElI ","docker":{"container_id":"728df538617fa22d57cac9d28b79079398171cdd1f24897aec60fe0e43a8135e"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"test-logging","pod_name":"centos-logtest-76qb9","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"5f049a1e-0586-4493-98a1-9329b8cf1d7b","pod_ip":"10.131.0.174","host":"worker-1","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"5a1de092-661d-4860-b096-cf853ef15b38","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:09:08.008485+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":671,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_app","secure":"true","syslog":"east"}},"viaq_msg_id":"ZWE5NmNmZDUtODU2YS00OTFjLWE3ZTAtYTExNzU2MzFmOWI3","log_type":"application"} Apr 1 08:09:09 worker-1 ocp47.22[myproc] {"@timestamp":"2025-04-01T08:09:08.323608110+00:00","message":"2025-04-01 08:09:08,313 - SVTLogger - INFO - centos-logtest-2jxtm : 333321 : VgUkScyIZ zUCWCrDxy lfHgVb93A H9ujF2A0K 6kuVChDnI 0t18rVoGU jCLQBAShI dohYUbtkR Ni9lGhAlt gDrhOyELc QEjlCfT5w jtlbpta22 3fybZNOtK FUwwgooMw ram8Cu7LD RAizcGkaK wtBFsrIgq mECw2O1CK 6Y4xI1AxX QJx3tKj4y ","docker":{"container_id":"f1ccdc3a273ba9353d7248786f079ea9e3ab6630f2a0ca75d20742ff1ba8cbdb"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"include-app","pod_name":"centos-logtest-2jxtm","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"c9523cb3-4a8c-4582-9d2a-ade46a914d0f","pod_ip":"10.131.0.179","host":"worker-1","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"5fa61797-a680-4391-b78c-6fddd25895dd","namespace_labels":{"kubernetes_io_metadata_name":"include-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:09:08.335445+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":672,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_app","secure":"true","syslog":"east"}},"viaq_msg_id":"YmJlY2I5YWMtZTU2YS00ZDAxLWEyZWUtZTNhOGYxNTlkNWQ0","log_type":"application"} Apr 1 08:09:09 worker-1 ocp47.22[myproc] {"@timestamp":"2025-04-01T08:09:09.003998593+00:00","message":"2025-04-01 08:09:09,003 - SVTLogger - INFO - centos-logtest-76qb9 : 333331 : vRbDfq1QU PJPUR4Hdp hH3OaPxCv bWmWEHEWa YjNzvWMhq nIaGsTX0O JdCXdYart RvxtSNW8m o19v45lVa yfjE1b5x3 or29hvNAM KpaFnINVU 6SWegLOIf SLSXVzt15 BNMU5DKjW VtclrkjmB hYbKkQHc1 GVWEqRSFL MM6y8mSzO d9wmAUElI ","docker":{"container_id":"728df538617fa22d57cac9d28b79079398171cdd1f24897aec60fe0e43a8135e"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"test-logging","pod_name":"centos-logtest-76qb9","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"5f049a1e-0586-4493-98a1-9329b8cf1d7b","pod_ip":"10.131.0.174","host":"worker-1","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"5a1de092-661d-4860-b096-cf853ef15b38","namespace_labels":{"kubernetes_io_metadata_name":"test-logging","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:09:09.005563+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":673,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_app","secure":"true","syslog":"east"}},"viaq_msg_id":"ZGY5ZjAzMmQtMzBkNS00OWM2LWJlYjEtYjk4YmNhMzIxYjQx","log_type":"application"} Apr 1 08:09:09 worker-1 ocp47.22[myproc] {"@timestamp":"2025-04-01T08:09:09.003998301+00:00","message":"2025-04-01 08:09:09,003 - SVTLogger - INFO - centos-logtest-gjd47 : 333341 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu ","docker":{"container_id":"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"exclude-app","pod_name":"centos-logtest-gjd47","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"e522a963-f9e7-40e6-a502-fcaa5e58614f","pod_ip":"10.131.0.175","host":"worker-1","labels":{"run":"centos-logtest","test":"centos-logtest"},"master_url":"https://kubernetes.default.svc","namespace_id":"4566a766-d384-4fb9-8430-1c31bd2242b5","namespace_labels":{"kubernetes_io_metadata_name":"exclude-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:09:09.008118+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":674,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_app","secure":"true","syslog":"east"}},"viaq_msg_id":"NGY5OWY1NjMtZmRhYi00MDE0LTgzZmMtZWE3NDM0MjI1M2Zl","log_type":"application"} ``` Infrastructure logs: ``` Apr 1 08:16:38 master-0 ocp47.22[myproc] {"@timestamp":"2025-04-01T08:16:38.040817588+00:00","message":"I0401 08:16:38.040774 1 ovn_kubernetes.go:2135] daemonset openshift-network-node-identity/network-node-identity rollout complete","docker":{"container_id":"fef4ae16b9a2b0ce73c53a7e69a41b1634c7e8520998c239cfd0c8def5657685"},"kubernetes":{"container_name":"network-operator","namespace_name":"openshift-network-operator","pod_name":"network-operator-5676fb67b6-j22x9","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70aa7c10147eb3945e6d60a11af1b73cfbbcdfa4ce9a70636160be134c2e96b0","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70aa7c10147eb3945e6d60a11af1b73cfbbcdfa4ce9a70636160be134c2e96b0","pod_id":"1f1f5899-a045-45db-9f3f-ac486a839ba1","pod_ip":"10.20.184.44","host":"master-0","labels":{"name":"network-operator","pod-template-hash":"5676fb67b6"},"master_url":"https://kubernetes.default.svc","namespace_id":"56992b16-446c-43f6-937b-ca0e0e9c42af","namespace_labels":{"kubernetes_io_metadata_name":"openshift-network-operator","name":"openshift-network-operator","olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["name=network-operator","pod-template-hash=5676fb67b6"]},"level":"info","hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.44","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:16:38.042739+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":4709,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_infra","secure":"true","syslog":"east"}},"viaq_msg_id":"NGRiNTViYzgtMTY0NS00ZmE1LWI0YzUtYjE0YzcxYmIwM2Ri","log_type":"infrastructure"} Apr 1 08:16:38 master-0 ocp47.22[myproc] {"@timestamp":"2025-04-01T08:16:38.051153799+00:00","message":"I0401 08:16:38.051096 1 log.go:194] Render phase done, rendered 110 objects","docker":{"container_id":"fef4ae16b9a2b0ce73c53a7e69a41b1634c7e8520998c239cfd0c8def5657685"},"kubernetes":{"container_name":"network-operator","namespace_name":"openshift-network-operator","pod_name":"network-operator-5676fb67b6-j22x9","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70aa7c10147eb3945e6d60a11af1b73cfbbcdfa4ce9a70636160be134c2e96b0","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70aa7c10147eb3945e6d60a11af1b73cfbbcdfa4ce9a70636160be134c2e96b0","pod_id":"1f1f5899-a045-45db-9f3f-ac486a839ba1","pod_ip":"10.20.184.44","host":"master-0","labels":{"name":"network-operator","pod-template-hash":"5676fb67b6"},"master_url":"https://kubernetes.default.svc","namespace_id":"56992b16-446c-43f6-937b-ca0e0e9c42af","namespace_labels":{"kubernetes_io_metadata_name":"openshift-network-operator","name":"openshift-network-operator","olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["name=network-operator","pod-template-hash=5676fb67b6"]},"level":"info","hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.44","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:16:38.052043+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":4710,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_infra","secure":"true","syslog":"east"}},"viaq_msg_id":"Y2YzZGJlMWQtZGMwYy00YmQ2LTkxZDAtYmUyZjQ3NTRhN2U1","log_type":"infrastructure"} Apr 1 08:16:38 master-0 ocp47.22[myproc] {"@timestamp":"2025-04-01T08:16:38.093686739+00:00","message":"I0401 08:16:38.093586 1 log.go:194] reconciling (/v1, Kind=ConfigMap) openshift-network-operator/applied-cluster","docker":{"container_id":"fef4ae16b9a2b0ce73c53a7e69a41b1634c7e8520998c239cfd0c8def5657685"},"kubernetes":{"container_name":"network-operator","namespace_name":"openshift-network-operator","pod_name":"network-operator-5676fb67b6-j22x9","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70aa7c10147eb3945e6d60a11af1b73cfbbcdfa4ce9a70636160be134c2e96b0","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70aa7c10147eb3945e6d60a11af1b73cfbbcdfa4ce9a70636160be134c2e96b0","pod_id":"1f1f5899-a045-45db-9f3f-ac486a839ba1","pod_ip":"10.20.184.44","host":"master-0","labels":{"name":"network-operator","pod-template-hash":"5676fb67b6"},"master_url":"https://kubernetes.default.svc","namespace_id":"56992b16-446c-43f6-937b-ca0e0e9c42af","namespace_labels":{"kubernetes_io_metadata_name":"openshift-network-operator","name":"openshift-network-operator","olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f":"","openshift_io_cluster-monitoring":"true","openshift_io_run-level":"0","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["name=network-operator","pod-template-hash=5676fb67b6"]},"level":"info","hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.44","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:16:38.095359+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":4711,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_infra","secure":"true","syslog":"east"}},"viaq_msg_id":"MzFkZWExYTAtZTI4MS00YTdkLWJhZGEtYTUxNGIwYjBhZDc5","log_type":"infrastructure"} Apr 1 08:16:38 master-2 ocp47.22[myproc] {"@timestamp":"2025-04-01T08:16:37.146293479+00:00","message":"ts=2025-04-01T08:16:37.146Z caller=collector.go:169 level=error msg=\"collector failed\" name=fibrechannel duration_seconds=0.003952061 err=\"error obtaining FibreChannel class info: failed to read file \\\"/host/sys/class/fc_host/host1/symbolic_name\\\": open /host/sys/class/fc_host/host1/symbolic_name: no such file or directory\"","docker":{"container_id":"8f15f244ad807ab8a73792c18ed5387e602e7925b9863c0b6aed856c8438b1b7"},"kubernetes":{"container_name":"node-exporter","namespace_name":"openshift-monitoring","pod_name":"node-exporter-gdjml","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9816ed2e1057d2c0e1967bb7806baa2f46bb8f2772b5e9d2b9c5d20c31d93e5e","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9816ed2e1057d2c0e1967bb7806baa2f46bb8f2772b5e9d2b9c5d20c31d93e5e","pod_id":"cb1b0dda-896b-4013-bf99-52e2b1de352f","pod_ip":"10.20.184.131","host":"master-2","labels":{"app_kubernetes_io_component":"exporter","app_kubernetes_io_managed-by":"cluster-monitoring-operator","app_kubernetes_io_name":"node-exporter","app_kubernetes_io_part-of":"openshift-monitoring","app_kubernetes_io_version":"1.6.1","controller-revision-hash":"84544887c4","pod-template-generation":"1"},"master_url":"https://kubernetes.default.svc","namespace_id":"ac86bef9-a349-4fb7-bf2b-24c34b3c3d3c","namespace_labels":{"kubernetes_io_metadata_name":"openshift-monitoring","name":"openshift-monitoring","network_openshift_io_policy-group":"monitoring","olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f":"","openshift_io_cluster-monitoring":"true","pod-security_kubernetes_io_audit":"privileged","pod-security_kubernetes_io_enforce":"privileged","pod-security_kubernetes_io_warn":"privileged"},"flat_labels":["app_kubernetes_io_component=exporter","app_kubernetes_io_managed-by=cluster-monitoring-operator","app_kubernetes_io_name=node-exporter","app_kubernetes_io_part-of=openshift-monitoring","app_kubernetes_io_version=1_6_1","controller-revision-hash=84544887c4","pod-template-generation=1"]},"level":"error","hostname":"master-2","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.131","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:16:37.147800+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":607,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_infra","secure":"true","syslog":"east"}},"viaq_msg_id":"NDQ3YzZlODQtNWMwNS00YzY2LThlYzItZWZjNTIwMmJjNWRj","log_type":"infrastructure"} Apr 1 08:16:38 worker-1 ocp47.22[myproc] {"_RUNTIME_SCOPE":"system","_STREAM_ID":"af46a9628f4949c6b6dee2d8993a14f8","_SYSTEMD_INVOCATION_ID":"58cada7f5fcb4d8dae3b3cf705eccf44","systemd":{"t":{"BOOT_ID":"336bdc198a164d66ab409c5fe7c6727f","CAP_EFFECTIVE":"1ffffffffff","CMDLINE":"/usr/bin/crio","COMM":"crio","EXE":"/usr/bin/crio","GID":"0","MACHINE_ID":"583465373f314990b03b62ef8ac54f0a","PID":"2779","SELINUX_CONTEXT":"system_u:system_r:container_runtime_t:s0","STREAM_ID":"af46a9628f4949c6b6dee2d8993a14f8","SYSTEMD_CGROUP":"/system.slice/crio.service","SYSTEMD_INVOCATION_ID":"58cada7f5fcb4d8dae3b3cf705eccf44","SYSTEMD_SLICE":"system.slice","SYSTEMD_UNIT":"crio.service","TRANSPORT":"stdout","UID":"0"},"u":{"SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"crio"}},"level":"info","message":"time=\"2025-04-01 08:16:37.737000235Z\" level=warning msg=\"Found defunct process with PID 1834543 (init.sh)\"","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:16:37.949766+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":3611,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_infra","secure":"true","syslog":"east"}},"@timestamp":"2025-04-01T08:16:37.737140+00:00","viaq_msg_id":"MGNmOGY5ZTktOTc4ZC00MmJkLWIzMTgtMDdmNmFjNTRkZjQ2","log_type":"infrastructure"} ``` Audit logs: ``` Apr 1 08:20:01 master-0 ocp47.22[myproc] {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"46cf9e36-2f26-4285-9b42-b9784b5668d5","stage":"ResponseComplete","requestURI":"/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/worker-0?timeout=10s","verb":"update","user":{"username":"system:node:worker-0","groups":["system:nodes","system:authenticated"]},"sourceIPs":["10.20.178.79"],"userAgent":"kubelet/v1.27.16+03a907c (linux/ppc64le) kubernetes/994a6d8","objectRef":{"resource":"leases","namespace":"kube-node-lease","name":"worker-0","uid":"d6fe4d14-eb37-4ab1-bbf3-b681b3ee4330","apiGroup":"coordination.k8s.io","apiVersion":"v1","resourceVersion":"5230534"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-04-01T08:20:01.251685Z","stageTimestamp":"2025-04-01T08:20:01.265159Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":""},"@timestamp":"2025-04-01T08:20:01.251685Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.44","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:20:01.265828+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1989,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_audit","secure":"true","syslog":"east"}},"viaq_msg_id":"NDRmZDgzYWEtMjUzYS00ZDE0LWI0ZDEtYzRiMWQ5Y2FmOWY1","log_type":"audit"} Apr 1 08:20:01 master-0 ocp47.22[myproc] {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"52853d9e-7761-4a97-98f0-f7dc167d60d6","stage":"ResponseComplete","requestURI":"/apis/rbac.authorization.k8s.io/v1/namespaces/openshift-cluster-storage-operator/roles/prometheus","verb":"get","user":{"username":"system:serviceaccount:openshift-cluster-version:default","uid":"991ba301-5754-4597-a013-70b0a42ba57b","groups":["system:serviceaccounts","system:serviceaccounts:openshift-cluster-version","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["cluster-version-operator-86dc4548b4-fqtzx"],"authentication.kubernetes.io/pod-uid":["4ab7feb2-60f1-4143-ae52-e77a40d9e8e9"]}},"sourceIPs":["10.20.178.79"],"userAgent":"cluster-version-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"roles","namespace":"openshift-cluster-storage-operator","name":"prometheus","apiGroup":"rbac.authorization.k8s.io","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-04-01T08:20:01.297601Z","stageTimestamp":"2025-04-01T08:20:01.301361Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cluster-version-operator\" of ClusterRole \"cluster-admin\" to ServiceAccount \"default/openshift-cluster-version\""},"@timestamp":"2025-04-01T08:20:01.297601Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.44","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:20:01.302564+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1990,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_audit","secure":"true","syslog":"east"}},"viaq_msg_id":"YTliMjZiMGYtMDk4Ny00OTlkLWFhNmYtNmZlYWQzNDZmYjM3","log_type":"audit"} Apr 1 08:20:01 master-0 ocp47.22[myproc] {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"c942dfd4-9103-4398-8c13-1dea69026acc","stage":"ResponseComplete","requestURI":"/apis/rbac.authorization.k8s.io/v1/namespaces/openshift-network-node-identity/rolebindings/network-node-identity-leases","verb":"get","user":{"username":"system:serviceaccount:openshift-network-operator:default","uid":"5e0291bf-9b67-49c1-9a32-d7821ac7f4d9","groups":["system:serviceaccounts","system:serviceaccounts:openshift-network-operator","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["network-operator-5676fb67b6-j22x9"],"authentication.kubernetes.io/pod-uid":["1f1f5899-a045-45db-9f3f-ac486a839ba1"]}},"sourceIPs":["10.20.178.79"],"userAgent":"cluster-network-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"rolebindings","namespace":"openshift-network-node-identity","name":"network-node-identity-leases","apiGroup":"rbac.authorization.k8s.io","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-04-01T08:20:01.309367Z","stageTimestamp":"2025-04-01T08:20:01.313997Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"default-account-cluster-network-operator\" of ClusterRole \"cluster-admin\" to ServiceAccount \"default/openshift-network-operator\""},"@timestamp":"2025-04-01T08:20:01.309367Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.44","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:20:01.314566+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1991,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_audit","secure":"true","syslog":"east"}},"viaq_msg_id":"MjMzMTA4MzQtY2JjMC00ZDM2LTljMjUtZjY3N2ZiOGZhYWQ1","log_type":"audit"} Apr 1 08:20:01 master-0 ocp47.22[myproc] {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"25e74a32-63b4-4771-8168-b810dd2325e2","stage":"ResponseComplete","requestURI":"/apis/scheduling.k8s.io/v1/priorityclasses/openshift-user-critical","verb":"get","user":{"username":"system:serviceaccount:openshift-cluster-version:default","uid":"991ba301-5754-4597-a013-70b0a42ba57b","groups":["system:serviceaccounts","system:serviceaccounts:openshift-cluster-version","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["cluster-version-operator-86dc4548b4-fqtzx"],"authentication.kubernetes.io/pod-uid":["4ab7feb2-60f1-4143-ae52-e77a40d9e8e9"]}},"sourceIPs":["10.20.178.79"],"userAgent":"cluster-version-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"priorityclasses","name":"openshift-user-critical","apiGroup":"scheduling.k8s.io","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-04-01T08:20:01.346299Z","stageTimestamp":"2025-04-01T08:20:01.350857Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cluster-version-operator\" of ClusterRole \"cluster-admin\" to ServiceAccount \"default/openshift-cluster-version\""},"@timestamp":"2025-04-01T08:20:01.346299Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.44","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:20:01.351725+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1992,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_audit","secure":"true","syslog":"east"}},"viaq_msg_id":"NGY3NjNkOWUtZjdiOC00NTgwLWIxOTctMjg0M2EwZjg5MzZk","log_type":"audit"} Apr 1 08:20:01 master-0 ocp47.22[myproc] {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"2b2bde86-61c2-4d20-b0c5-c70fb45663d6","stage":"ResponseComplete","requestURI":"/apis/rbac.authorization.k8s.io/v1/namespaces/openshift-cluster-storage-operator/rolebindings/prometheus","verb":"get","user":{"username":"system:serviceaccount:openshift-cluster-version:default","uid":"991ba301-5754-4597-a013-70b0a42ba57b","groups":["system:serviceaccounts","system:serviceaccounts:openshift-cluster-version","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["cluster-version-operator-86dc4548b4-fqtzx"],"authentication.kubernetes.io/pod-uid":["4ab7feb2-60f1-4143-ae52-e77a40d9e8e9"]}},"sourceIPs":["10.20.178.79"],"userAgent":"cluster-version-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"rolebindings","namespace":"openshift-cluster-storage-operator","name":"prometheus","apiGroup":"rbac.authorization.k8s.io","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-04-01T08:20:01.400135Z","stageTimestamp":"2025-04-01T08:20:01.407330Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cluster-version-operator\" of ClusterRole \"cluster-admin\" to ServiceAccount \"default/openshift-cluster-version\""},"@timestamp":"2025-04-01T08:20:01.400135Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-0","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.44","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T08:20:01.408289+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1993,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"label":"5.8_414_audit","secure":"true","syslog":"east"}},"viaq_msg_id":"MWYxYmQ5MGItMzY5ZC00M2Q3LTg5ZWYtNTAwMDgzODFhOGYw","log_type":"audit"} ``` Elasticsearch: Application logs: ``` {"_index":"app-write","_type":"_doc","_id":"MjkxMWM2ZDgtNDBkNi00YjNlLTgyM2MtZjdhOTMxMWNkN2I0","_score":4.8358823E-5,"_source":{"@timestamp":"2025-04-01T08:18:14.803947656+00:00","message":"2025-04-01 08:18:14,803 - SVTLogger - INFO - centos-logtest-gjd47 : 333886 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu ","docker":{"container_id":"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"exclude-app","pod_name":"centos-logtest-gjd47","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"e522a963-f9e7-40e6-a502-fcaa5e58614f","pod_ip":"10.131.0.175","host":"worker-1","master_url":"https://kubernetes.default.svc","namespace_id":"4566a766-d384-4fb9-8430-1c31bd2242b5","namespace_labels":{"kubernetes_io_metadata_name":"exclude-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:11:49.235935+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1110,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-application"}},"viaq_msg_id":"MjkxMWM2ZDgtNDBkNi00YjNlLTgyM2MtZjdhOTMxMWNkN2I0","log_type":"application"}},{"_index":"app-write","_type":"_doc","_id":"ZTRmZmFjODQtNWEzNy00N2Q1LWFiNjYtNzk5Zjg5ZjM5N2M3","_score":4.8358823E-5,"_source":{"@timestamp":"2025-04-01T08:18:15.805575629+00:00","message":"2025-04-01 08:18:15,805 - SVTLogger - INFO - centos-logtest-gjd47 : 333887 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu ","docker":{"container_id":"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"exclude-app","pod_name":"centos-logtest-gjd47","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"e522a963-f9e7-40e6-a502-fcaa5e58614f","pod_ip":"10.131.0.175","host":"worker-1","master_url":"https://kubernetes.default.svc","namespace_id":"4566a766-d384-4fb9-8430-1c31bd2242b5","namespace_labels":{"kubernetes_io_metadata_name":"exclude-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:11:49.236103+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1111,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-application"}},"viaq_msg_id":"ZTRmZmFjODQtNWEzNy00N2Q1LWFiNjYtNzk5Zjg5ZjM5N2M3","log_type":"application"}},{"_index":"app-write","_type":"_doc","_id":"MTE3MGNiZDYtNzYyYS00NDFiLWE0NGEtZjA5MzdlNGIwNmFl","_score":4.8358823E-5,"_source":{"@timestamp":"2025-04-01T08:18:16.807000322+00:00","message":"2025-04-01 08:18:16,806 - SVTLogger - INFO - centos-logtest-gjd47 : 333888 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu ","docker":{"container_id":"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"exclude-app","pod_name":"centos-logtest-gjd47","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"e522a963-f9e7-40e6-a502-fcaa5e58614f","pod_ip":"10.131.0.175","host":"worker-1","master_url":"https://kubernetes.default.svc","namespace_id":"4566a766-d384-4fb9-8430-1c31bd2242b5","namespace_labels":{"kubernetes_io_metadata_name":"exclude-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:11:49.238768+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1112,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-application"}},"viaq_msg_id":"MTE3MGNiZDYtNzYyYS00NDFiLWE0NGEtZjA5MzdlNGIwNmFl","log_type":"application"}},{"_index":"app-write","_type":"_doc","_id":"NDQwM2RhNDUtZDUzNi00ZjJjLTlkMjktNThiMWMyODFhYzFl","_score":4.8358823E-5,"_source":{"@timestamp":"2025-04-01T08:18:17.808579368+00:00","message":"2025-04-01 08:18:17,808 - SVTLogger - INFO - centos-logtest-gjd47 : 333889 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu ","docker":{"container_id":"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"exclude-app","pod_name":"centos-logtest-gjd47","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"e522a963-f9e7-40e6-a502-fcaa5e58614f","pod_ip":"10.131.0.175","host":"worker-1","master_url":"https://kubernetes.default.svc","namespace_id":"4566a766-d384-4fb9-8430-1c31bd2242b5","namespace_labels":{"kubernetes_io_metadata_name":"exclude-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:11:49.238936+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1113,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-application"}},"viaq_msg_id":"NDQwM2RhNDUtZDUzNi00ZjJjLTlkMjktNThiMWMyODFhYzFl","log_type":"application"}},{"_index":"app-write","_type":"_doc","_id":"OGU3MDY2YzEtNWQ3MS00Mjc5LThiZGUtY2I4YzJmMmVhNjhh","_score":4.8358823E-5,"_source":{"@timestamp":"2025-04-01T08:18:18.810013983+00:00","message":"2025-04-01 08:18:18,809 - SVTLogger - INFO - centos-logtest-gjd47 : 333890 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu ","docker":{"container_id":"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"exclude-app","pod_name":"centos-logtest-gjd47","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"e522a963-f9e7-40e6-a502-fcaa5e58614f","pod_ip":"10.131.0.175","host":"worker-1","master_url":"https://kubernetes.default.svc","namespace_id":"4566a766-d384-4fb9-8430-1c31bd2242b5","namespace_labels":{"kubernetes_io_metadata_name":"exclude-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:11:49.239102+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1114,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-application"}},"viaq_msg_id":"OGU3MDY2YzEtNWQ3MS00Mjc5LThiZGUtY2I4YzJmMmVhNjhh","log_type":"application"}},{"_index":"app-write","_type":"_doc","_id":"ZTIyYTEzMmEtYzE0Ni00MGVlLWI4ZGUtNGM0M2ZiNDcwZDYw","_score":4.8358823E-5,"_source":{"@timestamp":"2025-04-01T08:18:19.811645690+00:00","message":"2025-04-01 08:18:19,811 - SVTLogger - INFO - centos-logtest-gjd47 : 333891 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu ","docker":{"container_id":"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"exclude-app","pod_name":"centos-logtest-gjd47","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"e522a963-f9e7-40e6-a502-fcaa5e58614f","pod_ip":"10.131.0.175","host":"worker-1","master_url":"https://kubernetes.default.svc","namespace_id":"4566a766-d384-4fb9-8430-1c31bd2242b5","namespace_labels":{"kubernetes_io_metadata_name":"exclude-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:11:49.239269+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1115,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-application"}},"viaq_msg_id":"ZTIyYTEzMmEtYzE0Ni00MGVlLWI4ZGUtNGM0M2ZiNDcwZDYw","log_type":"application"}},{"_index":"app-write","_type":"_doc","_id":"ZjdmZTMyYzItOTUxZC00OWE3LWFjMWUtOWUxZjc2YTY2MzEw","_score":4.8358823E-5,"_source":{"@timestamp":"2025-04-01T08:18:20.813350354+00:00","message":"2025-04-01 08:18:20,812 - SVTLogger - INFO - centos-logtest-gjd47 : 333892 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu ","docker":{"container_id":"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00"},"kubernetes":{"container_name":"centos-logtest","namespace_name":"exclude-app","pod_name":"centos-logtest-gjd47","container_image":"quay.io/pravin_dsilva/centos:logtest","container_image_id":"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912","pod_id":"e522a963-f9e7-40e6-a502-fcaa5e58614f","pod_ip":"10.131.0.175","host":"worker-1","master_url":"https://kubernetes.default.svc","namespace_id":"4566a766-d384-4fb9-8430-1c31bd2242b5","namespace_labels":{"kubernetes_io_metadata_name":"exclude-app","pod-security_kubernetes_io_audit":"restricted","pod-security_kubernetes_io_audit-version":"v1.24","pod-security_kubernetes_io_warn":"restricted","pod-security_kubernetes_io_warn-version":"v1.24"},"flat_labels":["run=centos-logtest","test=centos-logtest"]},"level":"info","hostname":"worker-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.151","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:11:49.239439+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":1116,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-application"}},"viaq_msg_id":"ZjdmZTMyYzItOTUxZC00OWE3LWFjMWUtOWUxZjc2YTY2MzEw","log_type":"application"}}]}} ``` Audit logs: ``` {"_index":"audit-write","_type":"_doc","_id":"ZDk1Yzc1ZjktYWNmYy00YWJlLWIyOGYtNTQ0MTdmMTVhZTU1","_score":6.035913E-4,"_source":{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"e5f40001-a90b-4057-854a-3e58f88d55f4","stage":"ResponseComplete","requestURI":"/apis/route.openshift.io/v1/namespaces/openshift-ingress-operator/routes?allowWatchBookmarks=true&resourceVersion=5270901&timeoutSeconds=505&watch=true","verb":"watch","user":{"username":"system:serviceaccount:openshift-ingress-operator:ingress-operator","groups":["system:serviceaccounts","system:serviceaccounts:openshift-ingress-operator","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["ingress-operator-78dc96b6d9-66rsq"],"authentication.kubernetes.io/pod-uid":["5cba8d5b-1671-45a7-841b-a8774bde2f1a"]}},"sourceIPs":["10.130.1.243","10.130.0.2"],"userAgent":"Go-http-client/2.0","objectRef":{"resource":"routes","namespace":"openshift-ingress-operator","apiGroup":"route.openshift.io","apiVersion":"v1"},"responseStatus":{"code":200},"requestReceivedTimestamp":"2025-04-01T10:04:16.226216Z","stageTimestamp":"2025-04-01T10:12:41.232701Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"openshift-ingress-operator\" of ClusterRole \"openshift-ingress-operator\" to ServiceAccount \"ingress-operator/openshift-ingress-operator\""},"@timestamp":"2025-04-01T10:04:16.226216Z","openshift_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:41.325433+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9734,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"ZDk1Yzc1ZjktYWNmYy00YWJlLWIyOGYtNTQ0MTdmMTVhZTU1","log_type":"audit"}},{"_index":"audit-write","_type":"_doc","_id":"MzI4NzQwYTYtNTUzYy00NmVkLWIzYzItODA3M2RlM2E4YmEz","_score":6.035913E-4,"_source":{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"4ca64444-9615-4202-961a-631dcc4f3f6b","stage":"ResponseStarted","requestURI":"/apis/route.openshift.io/v1/namespaces/openshift-ingress-operator/routes?allowWatchBookmarks=true&resourceVersion=5274413&timeoutSeconds=332&watch=true","verb":"watch","user":{"username":"system:serviceaccount:openshift-ingress-operator:ingress-operator","groups":["system:serviceaccounts","system:serviceaccounts:openshift-ingress-operator","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["ingress-operator-78dc96b6d9-66rsq"],"authentication.kubernetes.io/pod-uid":["5cba8d5b-1671-45a7-841b-a8774bde2f1a"]}},"sourceIPs":["10.130.1.243","10.130.0.2"],"userAgent":"Go-http-client/2.0","objectRef":{"resource":"routes","namespace":"openshift-ingress-operator","apiGroup":"route.openshift.io","apiVersion":"v1"},"responseStatus":{"code":200},"requestReceivedTimestamp":"2025-04-01T10:12:41.241155Z","stageTimestamp":"2025-04-01T10:12:41.247885Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"openshift-ingress-operator\" of ClusterRole \"openshift-ingress-operator\" to ServiceAccount \"ingress-operator/openshift-ingress-operator\""},"@timestamp":"2025-04-01T10:12:41.241155Z","openshift_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:41.325709+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9735,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"MzI4NzQwYTYtNTUzYy00NmVkLWIzYzItODA3M2RlM2E4YmEz","log_type":"audit"}},{"_index":"audit-write","_type":"_doc","_id":"ODI2YzU3MzctMzI5MC00NjA0LTljNmUtYzU4MzU0MGUxYmY3","_score":6.035913E-4,"_source":{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"5c60d280-57e9-4515-8805-f3d06044b7ff","stage":"ResponseComplete","requestURI":"/apis/rbac.authorization.k8s.io/v1/clusterroles/alertmanager-main","verb":"update","user":{"username":"system:serviceaccount:openshift-monitoring:cluster-monitoring-operator","uid":"ccf24b7e-9615-4f9d-9ee5-7c79ad4ac0b1","groups":["system:serviceaccounts","system:serviceaccounts:openshift-monitoring","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["cluster-monitoring-operator-6cd6c98f5f-vpk8n"],"authentication.kubernetes.io/pod-uid":["4e719d53-1608-4a89-a497-aa91c80f65c5"]}},"sourceIPs":["10.130.1.238"],"userAgent":"operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"clusterroles","name":"alertmanager-main","apiGroup":"rbac.authorization.k8s.io","apiVersion":"v1"},"responseStatus":{"code":200},"requestReceivedTimestamp":"2025-04-01T10:12:37.895450Z","stageTimestamp":"2025-04-01T10:12:37.977881Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cluster-monitoring-operator\" of ClusterRole \"cluster-monitoring-operator\" to ServiceAccount \"cluster-monitoring-operator/openshift-monitoring\""},"@timestamp":"2025-04-01T10:12:37.895450Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:38.920595+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9592,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"ODI2YzU3MzctMzI5MC00NjA0LTljNmUtYzU4MzU0MGUxYmY3","log_type":"audit"}},{"_index":"audit-write","_type":"_doc","_id":"ZTM3MDkwOTgtNzkxNC00ZGY2LTk4NmYtMjU5NGY1MzliMzEy","_score":6.035913E-4,"_source":{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"95c17146-40a6-4af0-bdb0-6c3e0f5be4d5","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/openshift-user-workload-monitoring/secrets/thanos-ruler-query-config","verb":"delete","user":{"username":"system:serviceaccount:openshift-monitoring:cluster-monitoring-operator","uid":"ccf24b7e-9615-4f9d-9ee5-7c79ad4ac0b1","groups":["system:serviceaccounts","system:serviceaccounts:openshift-monitoring","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["cluster-monitoring-operator-6cd6c98f5f-vpk8n"],"authentication.kubernetes.io/pod-uid":["4e719d53-1608-4a89-a497-aa91c80f65c5"]}},"sourceIPs":["10.130.1.238"],"userAgent":"operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"secrets","namespace":"openshift-user-workload-monitoring","name":"thanos-ruler-query-config","apiVersion":"v1"},"responseStatus":{"status":"Failure","message":"secrets \"thanos-ruler-query-config\" not found","reason":"NotFound","details":{"name":"thanos-ruler-query-config","kind":"secrets"},"code":404},"requestReceivedTimestamp":"2025-04-01T10:12:37.978673Z","stageTimestamp":"2025-04-01T10:12:37.982472Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cluster-monitoring-operator\" of ClusterRole \"cluster-monitoring-operator\" to ServiceAccount \"cluster-monitoring-operator/openshift-monitoring\""},"@timestamp":"2025-04-01T10:12:37.978673Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:38.920678+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9593,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"ZTM3MDkwOTgtNzkxNC00ZGY2LTk4NmYtMjU5NGY1MzliMzEy","log_type":"audit"}},{"_index":"audit-write","_type":"_doc","_id":"MzZiMTcyNjctNTYwOS00NTVhLWFlNTktZGY1MWE3MTgzMDA2","_score":6.035913E-4,"_source":{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"2a3581a8-e2c7-4cfc-89a0-0660615df252","stage":"ResponseComplete","requestURI":"/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/alertmanager-main","verb":"get","user":{"username":"system:serviceaccount:openshift-monitoring:cluster-monitoring-operator","uid":"ccf24b7e-9615-4f9d-9ee5-7c79ad4ac0b1","groups":["system:serviceaccounts","system:serviceaccounts:openshift-monitoring","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["cluster-monitoring-operator-6cd6c98f5f-vpk8n"],"authentication.kubernetes.io/pod-uid":["4e719d53-1608-4a89-a497-aa91c80f65c5"]}},"sourceIPs":["10.130.1.238"],"userAgent":"operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"clusterrolebindings","name":"alertmanager-main","apiGroup":"rbac.authorization.k8s.io","apiVersion":"v1"},"responseStatus":{"code":200},"requestReceivedTimestamp":"2025-04-01T10:12:37.979426Z","stageTimestamp":"2025-04-01T10:12:37.982809Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cluster-monitoring-operator\" of ClusterRole \"cluster-monitoring-operator\" to ServiceAccount \"cluster-monitoring-operator/openshift-monitoring\""},"@timestamp":"2025-04-01T10:12:37.979426Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:38.920736+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9594,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"MzZiMTcyNjctNTYwOS00NTVhLWFlNTktZGY1MWE3MTgzMDA2","log_type":"audit"}},{"_index":"audit-write","_type":"_doc","_id":"ZmNiMmZmZjQtMGIzOS00YmVlLWI0MjAtYWJlN2U3OWZlMjgw","_score":6.035913E-4,"_source":{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"61285186-fb36-4e9d-afe8-e3ba56db627f","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/openshift-user-workload-monitoring/secrets/thanos-ruler-alertmanagers-config","verb":"delete","user":{"username":"system:serviceaccount:openshift-monitoring:cluster-monitoring-operator","uid":"ccf24b7e-9615-4f9d-9ee5-7c79ad4ac0b1","groups":["system:serviceaccounts","system:serviceaccounts:openshift-monitoring","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["cluster-monitoring-operator-6cd6c98f5f-vpk8n"],"authentication.kubernetes.io/pod-uid":["4e719d53-1608-4a89-a497-aa91c80f65c5"]}},"sourceIPs":["10.130.1.238"],"userAgent":"operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"secrets","namespace":"openshift-user-workload-monitoring","name":"thanos-ruler-alertmanagers-config","apiVersion":"v1"},"responseStatus":{"status":"Failure","message":"secrets \"thanos-ruler-alertmanagers-config\" not found","reason":"NotFound","details":{"name":"thanos-ruler-alertmanagers-config","kind":"secrets"},"code":404},"requestReceivedTimestamp":"2025-04-01T10:12:37.983938Z","stageTimestamp":"2025-04-01T10:12:37.988054Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cluster-monitoring-operator\" of ClusterRole \"cluster-monitoring-operator\" to ServiceAccount \"cluster-monitoring-operator/openshift-monitoring\""},"@timestamp":"2025-04-01T10:12:37.983938Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:38.920796+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9595,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"ZmNiMmZmZjQtMGIzOS00YmVlLWI0MjAtYWJlN2U3OWZlMjgw","log_type":"audit"}},{"_index":"audit-write","_type":"_doc","_id":"MTAwNGQ4OWItYzBhOS00Zjg5LTkyN2QtMGQ0Y2JmOWZhMzdi","_score":6.035913E-4,"_source":{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"a7d92821-ad1e-408b-829e-d19221492fbc","stage":"ResponseComplete","requestURI":"/apis/policy/v1/namespaces/openshift-monitoring/poddisruptionbudgets/monitoring-plugin","verb":"update","user":{"username":"system:serviceaccount:openshift-monitoring:cluster-monitoring-operator","uid":"ccf24b7e-9615-4f9d-9ee5-7c79ad4ac0b1","groups":["system:serviceaccounts","system:serviceaccounts:openshift-monitoring","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["cluster-monitoring-operator-6cd6c98f5f-vpk8n"],"authentication.kubernetes.io/pod-uid":["4e719d53-1608-4a89-a497-aa91c80f65c5"]}},"sourceIPs":["10.130.1.238"],"userAgent":"operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"poddisruptionbudgets","namespace":"openshift-monitoring","name":"monitoring-plugin","apiGroup":"policy","apiVersion":"v1","resourceVersion":"3338834"},"responseStatus":{"code":200},"requestReceivedTimestamp":"2025-04-01T10:12:37.980757Z","stageTimestamp":"2025-04-01T10:12:37.988232Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by RoleBinding \"cluster-monitoring-operator/openshift-monitoring\" of ClusterRole \"cluster-monitoring-operator-namespaced\" to ServiceAccount \"cluster-monitoring-operator/openshift-monitoring\""},"@timestamp":"2025-04-01T10:12:37.980757Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:38.920859+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9596,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"MTAwNGQ4OWItYzBhOS00Zjg5LTkyN2QtMGQ0Y2JmOWZhMzdi","log_type":"audit"}},{"_index":"audit-write","_type":"_doc","_id":"N2Y4OWE5MjAtNTI0OS00M2E0LWFlNzItMzk5NjVjZDFlZmIy","_score":6.035913E-4,"_source":{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"78e0775c-e0dd-45ec-9b44-0f417a23270a","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/openshift-monitoring/serviceaccounts/alertmanager-main","verb":"get","user":{"username":"system:serviceaccount:openshift-monitoring:cluster-monitoring-operator","uid":"ccf24b7e-9615-4f9d-9ee5-7c79ad4ac0b1","groups":["system:serviceaccounts","system:serviceaccounts:openshift-monitoring","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["cluster-monitoring-operator-6cd6c98f5f-vpk8n"],"authentication.kubernetes.io/pod-uid":["4e719d53-1608-4a89-a497-aa91c80f65c5"]}},"sourceIPs":["10.130.1.238"],"userAgent":"operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"serviceaccounts","namespace":"openshift-monitoring","name":"alertmanager-main","apiVersion":"v1"},"responseStatus":{"code":200},"requestReceivedTimestamp":"2025-04-01T10:12:37.984216Z","stageTimestamp":"2025-04-01T10:12:37.988758Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by RoleBinding \"cluster-monitoring-operator/openshift-monitoring\" of ClusterRole \"cluster-monitoring-operator-namespaced\" to ServiceAccount \"cluster-monitoring-operator/openshift-monitoring\""},"@timestamp":"2025-04-01T10:12:37.984216Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:38.920925+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9597,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"N2Y4OWE5MjAtNTI0OS00M2E0LWFlNzItMzk5NjVjZDFlZmIy","log_type":"audit"}},{"_index":"audit-write","_type":"_doc","_id":"YjFmOGIyNzItNTUwOS00OWJkLWI1NmUtZmFhYjI0Yzc5MzZl","_score":6.035913E-4,"_source":{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"ff91db63-5368-4d97-8c57-3141a8f41617","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/openshift-monitoring/secrets/alertmanager-main-proxy","verb":"get","user":{"username":"system:serviceaccount:openshift-monitoring:cluster-monitoring-operator","uid":"ccf24b7e-9615-4f9d-9ee5-7c79ad4ac0b1","groups":["system:serviceaccounts","system:serviceaccounts:openshift-monitoring","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["cluster-monitoring-operator-6cd6c98f5f-vpk8n"],"authentication.kubernetes.io/pod-uid":["4e719d53-1608-4a89-a497-aa91c80f65c5"]}},"sourceIPs":["10.130.1.238"],"userAgent":"operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"secrets","namespace":"openshift-monitoring","name":"alertmanager-main-proxy","apiVersion":"v1"},"responseStatus":{"code":200},"requestReceivedTimestamp":"2025-04-01T10:12:37.991374Z","stageTimestamp":"2025-04-01T10:12:37.994397Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cluster-monitoring-operator\" of ClusterRole \"cluster-monitoring-operator\" to ServiceAccount \"cluster-monitoring-operator/openshift-monitoring\""},"@timestamp":"2025-04-01T10:12:37.991374Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:38.920986+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9598,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"YjFmOGIyNzItNTUwOS00OWJkLWI1NmUtZmFhYjI0Yzc5MzZl","log_type":"audit"}}]}} ``` Infrastructure logs: ``` {"_index":"infra-write","_type":"_doc","_id":"MDk3YzdlZjEtYTJjMC00NjBmLThhNDUtOGMyZmQ1ZTZlYjk1","_score":0.71306396,"_source":{"@timestamp":"2025-04-01T08:32:58.943033800+00:00","message":"I0401 08:32:58.942922 1 httplog.go:132] \"HTTP\" verb=\"GET\" URI=\"/metrics\" latency=\"5.560956ms\" userAgent=\"Prometheus/2.46.0\" audit-ID=\"8afe35e8-bb01-486a-8475-ce1e22acc72b\" srcIP=\"10.131.0.168:50296\" resp=200","docker":{"container_id":"853e159c74f9a08498faacdabe2d987b6bc3bcae16e930fabb2a1c01e39423a0"},"kubernetes":{"container_name":"insights-operator","namespace_name":"openshift-insights","pod_name":"insights-operator-85659d5f6d-tkt6f","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","pod_id":"00717466-e3ff-480f-9646-e7357eaa82e3","pod_ip":"10.130.1.219","host":"master-1","master_url":"https://kubernetes.default.svc","namespace_id":"13e96902-1834-48ea-a339-45c2b9e43a84","namespace_labels":{"kubernetes_io_metadata_name":"openshift-insights","name":"openshift-insights","openshift_io_cluster-monitoring":"true"},"flat_labels":["app=insights-operator","pod-template-hash=85659d5f6d"]},"level":"info","hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:08.817003+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":3446,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-infrastructure"}},"viaq_msg_id":"MDk3YzdlZjEtYTJjMC00NjBmLThhNDUtOGMyZmQ1ZTZlYjk1","log_type":"infrastructure"}},{"_index":"infra-write","_type":"_doc","_id":"NmZjMzE4NGQtODMwNi00MzVhLTk4NmItN2ViZjYwYTZlM2E3","_score":0.71306396,"_source":{"@timestamp":"2025-04-01T08:33:00.550438527+00:00","message":"I0401 08:33:00.550314 1 reflector.go:788] github.com/openshift/client-go/config/informers/externalversions/factory.go:101: Watch close - *v1.FeatureGate total 8 items received","docker":{"container_id":"853e159c74f9a08498faacdabe2d987b6bc3bcae16e930fabb2a1c01e39423a0"},"kubernetes":{"container_name":"insights-operator","namespace_name":"openshift-insights","pod_name":"insights-operator-85659d5f6d-tkt6f","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","pod_id":"00717466-e3ff-480f-9646-e7357eaa82e3","pod_ip":"10.130.1.219","host":"master-1","master_url":"https://kubernetes.default.svc","namespace_id":"13e96902-1834-48ea-a339-45c2b9e43a84","namespace_labels":{"kubernetes_io_metadata_name":"openshift-insights","name":"openshift-insights","openshift_io_cluster-monitoring":"true"},"flat_labels":["app=insights-operator","pod-template-hash=85659d5f6d"]},"level":"info","hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:08.817150+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":3447,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-infrastructure"}},"viaq_msg_id":"NmZjMzE4NGQtODMwNi00MzVhLTk4NmItN2ViZjYwYTZlM2E3","log_type":"infrastructure"}},{"_index":"infra-write","_type":"_doc","_id":"YmM5NzgyODAtYzk1NC00MzhmLWE3NmUtOTYyZjRhYzc4M2Rm","_score":0.71306396,"_source":{"@timestamp":"2025-04-01T08:33:01.429108035+00:00","message":"I0401 08:33:01.428991 1 controller.go:444] The operator is healthy","docker":{"container_id":"853e159c74f9a08498faacdabe2d987b6bc3bcae16e930fabb2a1c01e39423a0"},"kubernetes":{"container_name":"insights-operator","namespace_name":"openshift-insights","pod_name":"insights-operator-85659d5f6d-tkt6f","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","pod_id":"00717466-e3ff-480f-9646-e7357eaa82e3","pod_ip":"10.130.1.219","host":"master-1","master_url":"https://kubernetes.default.svc","namespace_id":"13e96902-1834-48ea-a339-45c2b9e43a84","namespace_labels":{"kubernetes_io_metadata_name":"openshift-insights","name":"openshift-insights","openshift_io_cluster-monitoring":"true"},"flat_labels":["app=insights-operator","pod-template-hash=85659d5f6d"]},"level":"info","hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:08.817280+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":3448,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-infrastructure"}},"viaq_msg_id":"YmM5NzgyODAtYzk1NC00MzhmLWE3NmUtOTYyZjRhYzc4M2Rm","log_type":"infrastructure"}},{"_index":"infra-write","_type":"_doc","_id":"MzdjY2E4ZTctMDM0Yy00ZjFhLTkzZWQtMjc5NmRiMjJjYzU1","_score":0.71306396,"_source":{"@timestamp":"2025-04-01T08:33:01.429234189+00:00","message":"I0401 08:33:01.429134 1 controller.go:325] No status update necessary, objects are identical","docker":{"container_id":"853e159c74f9a08498faacdabe2d987b6bc3bcae16e930fabb2a1c01e39423a0"},"kubernetes":{"container_name":"insights-operator","namespace_name":"openshift-insights","pod_name":"insights-operator-85659d5f6d-tkt6f","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","pod_id":"00717466-e3ff-480f-9646-e7357eaa82e3","pod_ip":"10.130.1.219","host":"master-1","master_url":"https://kubernetes.default.svc","namespace_id":"13e96902-1834-48ea-a339-45c2b9e43a84","namespace_labels":{"kubernetes_io_metadata_name":"openshift-insights","name":"openshift-insights","openshift_io_cluster-monitoring":"true"},"flat_labels":["app=insights-operator","pod-template-hash=85659d5f6d"]},"level":"info","hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:08.817459+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":3449,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-infrastructure"}},"viaq_msg_id":"MzdjY2E4ZTctMDM0Yy00ZjFhLTkzZWQtMjc5NmRiMjJjYzU1","log_type":"infrastructure"}},{"_index":"infra-write","_type":"_doc","_id":"NTc4YjA1Y2MtODg5NS00OWNhLWIxMTUtNWJmNTk2YmZhNTRh","_score":0.71306396,"_source":{"@timestamp":"2025-04-01T08:33:09.685934055+00:00","message":"I0401 08:33:09.685809 1 reflector.go:788] k8s.io/apiserver/pkg/server/dynamiccertificates/configmap_cafile_content.go:206: Watch close - *v1.ConfigMap total 8 items received","docker":{"container_id":"853e159c74f9a08498faacdabe2d987b6bc3bcae16e930fabb2a1c01e39423a0"},"kubernetes":{"container_name":"insights-operator","namespace_name":"openshift-insights","pod_name":"insights-operator-85659d5f6d-tkt6f","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","pod_id":"00717466-e3ff-480f-9646-e7357eaa82e3","pod_ip":"10.130.1.219","host":"master-1","master_url":"https://kubernetes.default.svc","namespace_id":"13e96902-1834-48ea-a339-45c2b9e43a84","namespace_labels":{"kubernetes_io_metadata_name":"openshift-insights","name":"openshift-insights","openshift_io_cluster-monitoring":"true"},"flat_labels":["app=insights-operator","pod-template-hash=85659d5f6d"]},"level":"info","hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:08.817646+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":3450,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-infrastructure"}},"viaq_msg_id":"NTc4YjA1Y2MtODg5NS00OWNhLWIxMTUtNWJmNTk2YmZhNTRh","log_type":"infrastructure"}},{"_index":"infra-write","_type":"_doc","_id":"NDg1MDVmOTktYTdkOS00OTI3LWI5MDMtM2M1MGQ2OTFjNWM1","_score":0.71306396,"_source":{"@timestamp":"2025-04-01T08:33:16.481404867+00:00","message":"I0401 08:33:16.481295 1 httplog.go:132] \"HTTP\" verb=\"GET\" URI=\"/metrics\" latency=\"6.466802ms\" userAgent=\"Prometheus/2.46.0\" audit-ID=\"1a9d15b9-d4e5-4023-a8b0-409e60b77edc\" srcIP=\"10.128.3.6:57078\" resp=200","docker":{"container_id":"853e159c74f9a08498faacdabe2d987b6bc3bcae16e930fabb2a1c01e39423a0"},"kubernetes":{"container_name":"insights-operator","namespace_name":"openshift-insights","pod_name":"insights-operator-85659d5f6d-tkt6f","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","pod_id":"00717466-e3ff-480f-9646-e7357eaa82e3","pod_ip":"10.130.1.219","host":"master-1","master_url":"https://kubernetes.default.svc","namespace_id":"13e96902-1834-48ea-a339-45c2b9e43a84","namespace_labels":{"kubernetes_io_metadata_name":"openshift-insights","name":"openshift-insights","openshift_io_cluster-monitoring":"true"},"flat_labels":["app=insights-operator","pod-template-hash=85659d5f6d"]},"level":"info","hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:08.817803+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":3451,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-infrastructure"}},"viaq_msg_id":"NDg1MDVmOTktYTdkOS00OTI3LWI5MDMtM2M1MGQ2OTFjNWM1","log_type":"infrastructure"}},{"_index":"infra-write","_type":"_doc","_id":"NmY1MzE0OTYtZDQ2MC00YTgxLWExOTUtYmFiN2Q5N2NjMDFj","_score":0.71306396,"_source":{"@timestamp":"2025-04-01T08:33:28.945122583+00:00","message":"I0401 08:33:28.945005 1 httplog.go:132] \"HTTP\" verb=\"GET\" URI=\"/metrics\" latency=\"6.608558ms\" userAgent=\"Prometheus/2.46.0\" audit-ID=\"c8a032a2-32d2-4fa7-84c5-f03107a44f9a\" srcIP=\"10.131.0.168:50296\" resp=200","docker":{"container_id":"853e159c74f9a08498faacdabe2d987b6bc3bcae16e930fabb2a1c01e39423a0"},"kubernetes":{"container_name":"insights-operator","namespace_name":"openshift-insights","pod_name":"insights-operator-85659d5f6d-tkt6f","container_image":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","container_image_id":"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6601becd09142d7b8c24a40a5034bdc683dd6627b2383a2d66373e716773e53f","pod_id":"00717466-e3ff-480f-9646-e7357eaa82e3","pod_ip":"10.130.1.219","host":"master-1","master_url":"https://kubernetes.default.svc","namespace_id":"13e96902-1834-48ea-a339-45c2b9e43a84","namespace_labels":{"kubernetes_io_metadata_name":"openshift-insights","name":"openshift-insights","openshift_io_cluster-monitoring":"true"},"flat_labels":["app=insights-operator","pod-template-hash=85659d5f6d"]},"level":"info","hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:08.817963+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":3452,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-infrastructure"}},"viaq_msg_id":"NmY1MzE0OTYtZDQ2MC00YTgxLWExOTUtYmFiN2Q5N2NjMDFj","log_type":"infrastructure"}}]}} ``` Kafka: Application logs: ``` ``` Audit logs: ``` {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"8d27896e-2d48-4058-859f-dc21d474ad1d","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/openshift-monitoring/secrets/prometheus-operator-kube-rbac-proxy-config","verb":"get","user":{"username":"system:serviceaccount:openshift-monitoring:cluster-monitoring-operator","uid":"ccf24b7e-9615-4f9d-9ee5-7c79ad4ac0b1","groups":["system:serviceaccounts","system:serviceaccounts:openshift-monitoring","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["cluster-monitoring-operator-6cd6c98f5f-vpk8n"],"authentication.kubernetes.io/pod-uid":["4e719d53-1608-4a89-a497-aa91c80f65c5"]}},"sourceIPs":["10.130.1.238"],"userAgent":"operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"secrets","namespace":"openshift-monitoring","name":"prometheus-operator-kube-rbac-proxy-config","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-04-01T10:12:34.617468Z","stageTimestamp":"2025-04-01T10:12:34.621479Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cluster-monitoring-operator\" of ClusterRole \"cluster-monitoring-operator\" to ServiceAccount \"cluster-monitoring-operator/openshift-monitoring\""},"@timestamp":"2025-04-01T10:12:34.617468Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:38.904964+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9357,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"ZmM2MzBlZjgtYTYyNi00OWUwLTlkN2ItOTU5MDM2ZjcxN2Vj","log_type":"audit"} {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"6a3bfb79-c245-416c-a304-feb86209ac3e","stage":"ResponseComplete","requestURI":"/apis/apps/v1/namespaces/openshift-monitoring/deployments/prometheus-operator","verb":"get","user":{"username":"system:serviceaccount:openshift-monitoring:cluster-monitoring-operator","uid":"ccf24b7e-9615-4f9d-9ee5-7c79ad4ac0b1","groups":["system:serviceaccounts","system:serviceaccounts:openshift-monitoring","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["cluster-monitoring-operator-6cd6c98f5f-vpk8n"],"authentication.kubernetes.io/pod-uid":["4e719d53-1608-4a89-a497-aa91c80f65c5"]}},"sourceIPs":["10.130.1.238"],"userAgent":"operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"deployments","namespace":"openshift-monitoring","name":"prometheus-operator","apiGroup":"apps","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-04-01T10:12:34.623754Z","stageTimestamp":"2025-04-01T10:12:34.630102Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by RoleBinding \"cluster-monitoring-operator/openshift-monitoring\" of ClusterRole \"cluster-monitoring-operator-namespaced\" to ServiceAccount \"cluster-monitoring-operator/openshift-monitoring\""},"@timestamp":"2025-04-01T10:12:34.623754Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:38.905023+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9358,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"NGNiNjQ0OGUtOGEzMi00MmMxLThjYzQtMGQzMjE4NTkwYTNh","log_type":"audit"} {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"3dc11224-a329-4f7d-b5b7-9ebe48cd156e","stage":"ResponseComplete","requestURI":"/apis/apps/v1/namespaces/openshift-monitoring/deployments/prometheus-operator","verb":"update","user":{"username":"system:serviceaccount:openshift-monitoring:cluster-monitoring-operator","uid":"ccf24b7e-9615-4f9d-9ee5-7c79ad4ac0b1","groups":["system:serviceaccounts","system:serviceaccounts:openshift-monitoring","system:authenticated"],"extra":{"authentication.kubernetes.io/pod-name":["cluster-monitoring-operator-6cd6c98f5f-vpk8n"],"authentication.kubernetes.io/pod-uid":["4e719d53-1608-4a89-a497-aa91c80f65c5"]}},"sourceIPs":["10.130.1.238"],"userAgent":"operator/v0.0.0 (linux/ppc64le) kubernetes/$Format","objectRef":{"resource":"deployments","namespace":"openshift-monitoring","name":"prometheus-operator","apiGroup":"apps","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-04-01T10:12:34.632468Z","stageTimestamp":"2025-04-01T10:12:34.647443Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by RoleBinding \"cluster-monitoring-operator/openshift-monitoring\" of ClusterRole \"cluster-monitoring-operator-namespaced\" to ServiceAccount \"cluster-monitoring-operator/openshift-monitoring\""},"@timestamp":"2025-04-01T10:12:34.632468Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:38.905092+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9359,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"MWRjMjYxNTgtMDNkMS00NDJlLWE4ZTYtODQ0OTBjZGIzOGMz","log_type":"audit"} {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"6e8f7283-4aa1-4e06-a78d-5272e71ff9d1","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/default","verb":"get","user":{"username":"system:apiserver","uid":"dc9bcc54-a9f5-451b-8676-c7c745761f5f","groups":["system:masters"]},"sourceIPs":["::1"],"userAgent":"kube-apiserver/v1.27.16+03a907c (linux/ppc64le) kubernetes/994a6d8","objectRef":{"resource":"namespaces","namespace":"default","name":"default","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-04-01T10:12:34.868859Z","stageTimestamp":"2025-04-01T10:12:34.873062Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":""},"@timestamp":"2025-04-01T10:12:34.868859Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:38.905153+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9360,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"ZTBiM2FjZjgtZjRlMC00MjU0LTkxOWUtMzViNjhjNjdlMjEy","log_type":"audit"} {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"3c56ceef-a182-4a74-acf6-0c564f451671","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/default/services/kubernetes","verb":"get","user":{"username":"system:apiserver","uid":"dc9bcc54-a9f5-451b-8676-c7c745761f5f","groups":["system:masters"]},"sourceIPs":["::1"],"userAgent":"kube-apiserver/v1.27.16+03a907c (linux/ppc64le) kubernetes/994a6d8","objectRef":{"resource":"services","namespace":"default","name":"kubernetes","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-04-01T10:12:34.874227Z","stageTimestamp":"2025-04-01T10:12:34.878309Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":""},"@timestamp":"2025-04-01T10:12:34.874227Z","k8s_audit_level":"Metadata","message":null,"hostname":"master-1","pipeline_metadata":{"collector":{"ipaddr4":"10.20.184.206","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2025-04-01T10:12:38.905215+00:00","version":"1.16.2 1.6.0"}},"openshift":{"sequence":9361,"cluster_id":"657154e1-ef9c-426e-8079-d58eaee0a17b","labels":{"logs":"5.8_414-audit"}},"viaq_msg_id":"ODAxNmZjNzMtZDRmYi00NjdlLWIxNjktYzk2NzdlNmEzNjU0","log_type":"audit"} ``` Loki: Application logs: ``` {"ts":"2025-04-01T10:13:12.028607284Z","line":"{\"@timestamp\":\"2025-04-01T10:13:12.028607284+00:00\",\"message\":\"2025-04-01 10:13:12,028 - SVTLogger - INFO - centos-logtest-gjd47 : 340773 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu \",\"docker\":{\"container_id\":\"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00\"},\"kubernetes\":{\"container_name\":\"centos-logtest\",\"namespace_name\":\"exclude-app\",\"pod_name\":\"centos-logtest-gjd47\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_image_id\":\"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912\",\"pod_id\":\"e522a963-f9e7-40e6-a502-fcaa5e58614f\",\"pod_ip\":\"10.131.0.175\",\"host\":\"worker-1\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"4566a766-d384-4fb9-8430-1c31bd2242b5\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"exclude-app\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"flat_labels\":[\"run=centos-logtest\",\"test=centos-logtest\"]},\"level\":\"info\",\"hostname\":\"worker-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.151\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:12.140067+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":35446,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-application\"}},\"viaq_msg_id\":\"MGRkNGEzMzUtZjUzZi00MDkzLTk2ZTgtOTkzYjQxNDJlZTU0\",\"log_type\":\"application\",\"structured\":{}}"},{"ts":"2025-04-01T10:13:09.024750712Z","line":"{\"@timestamp\":\"2025-04-01T10:13:09.024750712+00:00\",\"message\":\"2025-04-01 10:13:09,024 - SVTLogger - INFO - centos-logtest-gjd47 : 340770 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu \",\"docker\":{\"container_id\":\"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00\"},\"kubernetes\":{\"container_name\":\"centos-logtest\",\"namespace_name\":\"exclude-app\",\"pod_name\":\"centos-logtest-gjd47\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_image_id\":\"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912\",\"pod_id\":\"e522a963-f9e7-40e6-a502-fcaa5e58614f\",\"pod_ip\":\"10.131.0.175\",\"host\":\"worker-1\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"4566a766-d384-4fb9-8430-1c31bd2242b5\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"exclude-app\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"flat_labels\":[\"run=centos-logtest\",\"test=centos-logtest\"]},\"level\":\"info\",\"hostname\":\"worker-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.151\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:09.028741+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":34998,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-application\"}},\"viaq_msg_id\":\"ZWQ5ZDNiZDQtOTlmOC00YjQ3LTllY2ItYzM0YTg2YmMxMTBi\",\"log_type\":\"application\",\"structured\":{}}"},{"ts":"2025-04-01T10:13:07.022131927Z","line":"{\"@timestamp\":\"2025-04-01T10:13:07.022131927+00:00\",\"message\":\"2025-04-01 10:13:07,021 - SVTLogger - INFO - centos-logtest-gjd47 : 340768 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu \",\"docker\":{\"container_id\":\"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00\"},\"kubernetes\":{\"container_name\":\"centos-logtest\",\"namespace_name\":\"exclude-app\",\"pod_name\":\"centos-logtest-gjd47\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_image_id\":\"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912\",\"pod_id\":\"e522a963-f9e7-40e6-a502-fcaa5e58614f\",\"pod_ip\":\"10.131.0.175\",\"host\":\"worker-1\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"4566a766-d384-4fb9-8430-1c31bd2242b5\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"exclude-app\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"flat_labels\":[\"run=centos-logtest\",\"test=centos-logtest\"]},\"level\":\"info\",\"hostname\":\"worker-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.151\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:07.026813+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":34459,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-application\"}},\"viaq_msg_id\":\"MGMxZDM2NDUtYjVkNS00ZDAxLWI3MWQtOGI5Mjk1MTcyODQ5\",\"log_type\":\"application\",\"structured\":{}}"},{"ts":"2025-04-01T10:13:05.019699814Z","line":"{\"@timestamp\":\"2025-04-01T10:13:05.019699814+00:00\",\"message\":\"2025-04-01 10:13:05,019 - SVTLogger - INFO - centos-logtest-gjd47 : 340766 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu \",\"docker\":{\"container_id\":\"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00\"},\"kubernetes\":{\"container_name\":\"centos-logtest\",\"namespace_name\":\"exclude-app\",\"pod_name\":\"centos-logtest-gjd47\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_image_id\":\"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912\",\"pod_id\":\"e522a963-f9e7-40e6-a502-fcaa5e58614f\",\"pod_ip\":\"10.131.0.175\",\"host\":\"worker-1\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"4566a766-d384-4fb9-8430-1c31bd2242b5\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"exclude-app\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"flat_labels\":[\"run=centos-logtest\",\"test=centos-logtest\"]},\"level\":\"info\",\"hostname\":\"worker-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.151\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:05.050548+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":33885,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-application\"}},\"viaq_msg_id\":\"YmY4N2I2ZjMtMDE3MC00YTVkLTk5MmYtNjZmMTUxYTAwMGU5\",\"log_type\":\"application\",\"structured\":{}}"},{"ts":"2025-04-01T10:13:03.016938324Z","line":"{\"@timestamp\":\"2025-04-01T10:13:03.016938324+00:00\",\"message\":\"2025-04-01 10:13:03,016 - SVTLogger - INFO - centos-logtest-gjd47 : 340764 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu \",\"docker\":{\"container_id\":\"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00\"},\"kubernetes\":{\"container_name\":\"centos-logtest\",\"namespace_name\":\"exclude-app\",\"pod_name\":\"centos-logtest-gjd47\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_image_id\":\"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912\",\"pod_id\":\"e522a963-f9e7-40e6-a502-fcaa5e58614f\",\"pod_ip\":\"10.131.0.175\",\"host\":\"worker-1\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"4566a766-d384-4fb9-8430-1c31bd2242b5\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"exclude-app\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"flat_labels\":[\"run=centos-logtest\",\"test=centos-logtest\"]},\"level\":\"info\",\"hostname\":\"worker-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.151\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:03.018322+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":33437,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-application\"}},\"viaq_msg_id\":\"NWQ5Zjc5ODctZTFkYS00YWYyLWExMWYtOTY1YWQxMTUyMDFm\",\"log_type\":\"application\",\"structured\":{}}"},{"ts":"2025-04-01T10:13:01.014034392Z","line":"{\"@timestamp\":\"2025-04-01T10:13:01.014034392+00:00\",\"message\":\"2025-04-01 10:13:01,013 - SVTLogger - INFO - centos-logtest-gjd47 : 340762 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu \",\"docker\":{\"container_id\":\"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00\"},\"kubernetes\":{\"container_name\":\"centos-logtest\",\"namespace_name\":\"exclude-app\",\"pod_name\":\"centos-logtest-gjd47\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_image_id\":\"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912\",\"pod_id\":\"e522a963-f9e7-40e6-a502-fcaa5e58614f\",\"pod_ip\":\"10.131.0.175\",\"host\":\"worker-1\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"4566a766-d384-4fb9-8430-1c31bd2242b5\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"exclude-app\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"flat_labels\":[\"run=centos-logtest\",\"test=centos-logtest\"]},\"level\":\"info\",\"hostname\":\"worker-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.151\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:01.019652+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":33010,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-application\"}},\"viaq_msg_id\":\"MDUxYzAzZjgtNmUxNS00OTU3LWFiZWQtOTk2NDE5YzU2MzNi\",\"log_type\":\"application\",\"structured\":{}}"},{"ts":"2025-04-01T10:12:59.011796089Z","line":"{\"@timestamp\":\"2025-04-01T10:12:59.011796089+00:00\",\"message\":\"2025-04-01 10:12:59,011 - SVTLogger - INFO - centos-logtest-gjd47 : 340760 : eBx9O2h2g nJWBd54q1 ceAgdMInP UQaTeQe2s QJzdplNK3 H6D0I8cUK VWU4mhxtj mXnAvcnPp syHxuSYMe q9OiNyiMz 4YRPmfqlX knJ6fUBF0 NvVHYuZw2 xrYEddKD5 Z5Df61jEq YkVEwcoSo iFPDOI1ed P5bmrmqzb wwYmxC5JM JobiXjTFu \",\"docker\":{\"container_id\":\"1073266bde0b262c2f83f95fd7a31568091db18f0a60b259b754c3bf29279e00\"},\"kubernetes\":{\"container_name\":\"centos-logtest\",\"namespace_name\":\"exclude-app\",\"pod_name\":\"centos-logtest-gjd47\",\"container_image\":\"quay.io/pravin_dsilva/centos:logtest\",\"container_image_id\":\"quay.io/pravin_dsilva/centos@sha256:38b54ceb7e1963086aa531fb5462b271939c2bb7a3868bfe67a9a0721a910912\",\"pod_id\":\"e522a963-f9e7-40e6-a502-fcaa5e58614f\",\"pod_ip\":\"10.131.0.175\",\"host\":\"worker-1\",\"labels\":{\"run\":\"centos-logtest\",\"test\":\"centos-logtest\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"4566a766-d384-4fb9-8430-1c31bd2242b5\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"exclude-app\",\"pod-security_kubernetes_io_audit\":\"restricted\",\"pod-security_kubernetes_io_audit-version\":\"v1.24\",\"pod-security_kubernetes_io_warn\":\"restricted\",\"pod-security_kubernetes_io_warn-version\":\"v1.24\"},\"flat_labels\":[\"run=centos-logtest\",\"test=centos-logtest\"]},\"level\":\"info\",\"hostname\":\"worker-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.151\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:12:59.032439+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":32591,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-application\"}},\"viaq_msg_id\":\"NWZmNDIyZmEtNWIwNC00ZTg0LWJlZDktMWYxMzFmZmVjMzRh\",\"log_type\":\"application\",\"structured\":{}}"}]}],"stats":{"summary":{"bytesProcessedPerSecond":143541145,"linesProcessedPerSecond":81133,"totalBytesProcessed":1132286,"totalLinesProcessed":640,"execTime":0.007888233,"queueTime":0.000041,"subqueries":0,"totalEntriesReturned":100,"splits":1,"shards":0},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0}}},"ingester":{"totalReached":1,"totalChunksMatched":9,"totalBatches":1,"totalLinesSent":100,"store":{"totalChunksRef":1,"totalChunksDownloaded":1,"chunksDownloadTime":142415,"chunk":{"headChunkBytes":219129,"headChunkLines":125,"decompressedBytes":913157,"decompressedLines":515,"compressedBytes":236178,"totalDuplicates":0}}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0}}}} ``` Audit logs: ``` {"ts":"2025-04-01T10:13:09.113826Z","line":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"0bed9840-422c-4cfd-b0cb-90789213f93b\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/coordination.k8s.io/v1/namespaces/openshift-kube-scheduler/leases/cert-recovery-controller-lock?timeout=1m47s\",\"verb\":\"get\",\"user\":{\"username\":\"system:serviceaccount:openshift-kube-scheduler:localhost-recovery-client\",\"uid\":\"ce8a0e42-5b5a-4b4f-a68f-a7b896b23d4e\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-kube-scheduler\",\"system:authenticated\"]},\"sourceIPs\":[\"::1\"],\"userAgent\":\"cluster-kube-scheduler-operator/v0.0.0 (linux/ppc64le) kubernetes/$Format\",\"objectRef\":{\"resource\":\"leases\",\"namespace\":\"openshift-kube-scheduler\",\"name\":\"cert-recovery-controller-lock\",\"apiGroup\":\"coordination.k8s.io\",\"apiVersion\":\"v1\"},\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2025-04-01T10:13:09.113826Z\",\"stageTimestamp\":\"2025-04-01T10:13:09.121957Z\",\"annotations\":{\"authentication.k8s.io/legacy-token\":\"system:serviceaccount:openshift-kube-scheduler:localhost-recovery-client\",\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:openshift:operator:kube-scheduler-recovery\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"localhost-recovery-client/openshift-kube-scheduler\\\"\"},\"@timestamp\":\"2025-04-01T10:13:09.113826Z\",\"k8s_audit_level\":\"Metadata\",\"message\":null,\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.206\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:09.160857+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":14544,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-audit\"}},\"viaq_msg_id\":\"NWQ0MTAwMWMtNmFkNi00N2ZhLThhODctMDk2M2MxOTBhN2Q4\",\"log_type\":\"audit\"}"},{"ts":"2025-04-01T10:13:08.334305Z","line":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"903d7497-7c9c-4ee2-9eca-c9215847f8ca\",\"stage\":\"ResponseStarted\",\"requestURI\":\"/api/v1/namespaces/openshift-monitoring/secrets?allowWatchBookmarks=true\u0026fieldSelector=metadata.name%3Dthanos-querier-dockercfg-2zkl9\u0026resourceVersion=5274560\u0026timeout=8m55s\u0026timeoutSeconds=535\u0026watch=true\",\"verb\":\"watch\",\"user\":{\"username\":\"system:node:worker-1\",\"groups\":[\"system:nodes\",\"system:authenticated\"]},\"sourceIPs\":[\"10.20.178.79\"],\"userAgent\":\"kubelet/v1.27.16+03a907c (linux/ppc64le) kubernetes/994a6d8\",\"objectRef\":{\"resource\":\"secrets\",\"namespace\":\"openshift-monitoring\",\"name\":\"thanos-querier-dockercfg-2zkl9\",\"apiVersion\":\"v1\"},\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2025-04-01T10:13:08.334305Z\",\"stageTimestamp\":\"2025-04-01T10:13:08.336111Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"\"},\"@timestamp\":\"2025-04-01T10:13:08.334305Z\",\"k8s_audit_level\":\"Metadata\",\"message\":null,\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.206\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:08.345682+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":14497,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-audit\"}},\"viaq_msg_id\":\"YWFiNWVhZjctOWJhZC00MTE1LWEyMTctN2EwMjM4NWQ2NTVh\",\"log_type\":\"audit\"}"},{"ts":"2025-04-01T10:13:08.196955Z","line":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"15b70a7e-9f2f-4aed-946b-f797605180fd\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/livez\",\"verb\":\"get\",\"user\":{\"username\":\"system:anonymous\",\"groups\":[\"system:unauthenticated\"]},\"sourceIPs\":[\"10.20.184.206\"],\"userAgent\":\"kube-probe/1.27\",\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2025-04-01T10:13:08.196955Z\",\"stageTimestamp\":\"2025-04-01T10:13:08.202606Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:public-info-viewer\\\" of ClusterRole \\\"system:public-info-viewer\\\" to Group \\\"system:unauthenticated\\\"\"},\"@timestamp\":\"2025-04-01T10:13:08.196955Z\",\"k8s_audit_level\":\"Metadata\",\"message\":null,\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.206\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:08.233128+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":14480,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-audit\"}},\"viaq_msg_id\":\"YTI1OWRkZTQtNGEwNS00OGMxLThlMGYtNzdhMTlkNjQzYTFi\",\"log_type\":\"audit\"}"},{"ts":"2025-04-01T10:13:08.193574Z","line":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"97f13f17-7281-41eb-835f-58b13b2797e0\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/authorization.k8s.io/v1/subjectaccessreviews?timeout=10s\",\"verb\":\"create\",\"user\":{\"username\":\"system:serviceaccount:openshift-authentication:oauth-openshift\",\"uid\":\"03bcf11e-caba-487f-9d8a-5d8b2b43babe\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-authentication\",\"system:authenticated\"],\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"oauth-openshift-947489556-xnjr5\"],\"authentication.kubernetes.io/pod-uid\":[\"6ca28649-bca0-4798-b576-529949f036de\"]}},\"sourceIPs\":[\"10.20.184.44\"],\"userAgent\":\"oauth-server/v0.0.0 (linux/ppc64le) kubernetes/$Format\",\"objectRef\":{\"resource\":\"subjectaccessreviews\",\"apiGroup\":\"authorization.k8s.io\",\"apiVersion\":\"v1\"},\"responseStatus\":{\"metadata\":{},\"code\":201},\"requestReceivedTimestamp\":\"2025-04-01T10:13:08.193574Z\",\"stageTimestamp\":\"2025-04-01T10:13:08.205046Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:openshift:openshift-authentication\\\" of ClusterRole \\\"cluster-admin\\\" to ServiceAccount \\\"oauth-openshift/openshift-authentication\\\"\"},\"@timestamp\":\"2025-04-01T10:13:08.193574Z\",\"k8s_audit_level\":\"Metadata\",\"message\":null,\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.206\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:08.233446+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":14481,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-audit\"}},\"viaq_msg_id\":\"NDY2MTAxNTYtZWEyZC00MDFjLWE3MzQtMGMxMGM2NmVjYTA4\",\"log_type\":\"audit\"}"},{"ts":"2025-04-01T10:13:05.58518Z","line":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"98279e62-0cb3-41c5-8bc4-3ba91f48dad8\",\"stage\":\"ResponseStarted\",\"requestURI\":\"/api/v1/namespaces/kube-system/configmaps?allowWatchBookmarks=true\u0026fieldSelector=metadata.name%3Dextension-apiserver-authentication\u0026resourceVersion=5274554\u0026timeout=6m15s\u0026timeoutSeconds=375\u0026watch=true\",\"verb\":\"watch\",\"user\":{\"username\":\"system:serviceaccount:openshift-route-controller-manager:route-controller-manager-sa\",\"uid\":\"b35825e0-5b2a-41d2-b514-181ef07b6fee\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:openshift-route-controller-manager\",\"system:authenticated\"],\"extra\":{\"authentication.kubernetes.io/pod-name\":[\"route-controller-manager-598ccfc464-bxzbx\"],\"authentication.kubernetes.io/pod-uid\":[\"fbc9c3b2-6232-475b-9b74-b97e646c6a2a\"]}},\"sourceIPs\":[\"10.130.1.213\"],\"userAgent\":\"route-controller-manager/v0.0.0 (linux/ppc64le) kubernetes/$Format\",\"objectRef\":{\"resource\":\"configmaps\",\"namespace\":\"kube-system\",\"name\":\"extension-apiserver-authentication\",\"apiVersion\":\"v1\"},\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2025-04-01T10:13:05.585180Z\",\"stageTimestamp\":\"2025-04-01T10:13:05.589610Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by RoleBinding \\\"authentication-reader-for-authenticated-users/kube-system\\\" of Role \\\"extension-apiserver-authentication-reader\\\" to Group \\\"system:authenticated\\\"\"},\"@timestamp\":\"2025-04-01T10:13:05.585180Z\",\"k8s_audit_level\":\"Metadata\",\"message\":null,\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.206\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:05.607171+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":14222,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-audit\"}},\"viaq_msg_id\":\"OGI3ZTI0ODctYWJkMC00OTI5LTg0NzAtNTIxYTA5MjZiNmI4\",\"log_type\":\"audit\"}"}]}],"stats":{"summary":{"bytesProcessedPerSecond":149188885,"linesProcessedPerSecond":83059,"totalBytesProcessed":2724777,"totalLinesProcessed":1517,"execTime":0.018263941,"queueTime":0.000046,"subqueries":0,"totalEntriesReturned":100,"splits":1,"shards":0},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0}}},"ingester":{"totalReached":1,"totalChunksMatched":3,"totalBatches":1,"totalLinesSent":100,"store":{"totalChunksRef":2,"totalChunksDownloaded":2,"chunksDownloadTime":325776,"chunk":{"headChunkBytes":157361,"headChunkLines":102,"decompressedBytes":2567416,"decompressedLines":1415,"compressedBytes":277433,"totalDuplicates":0}}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0}}}} ``` Infrastructure logs: ``` {"ts":"2025-04-01T10:13:03.924131846Z","line":"{\"@timestamp\":\"2025-04-01T10:13:03.924131846+00:00\",\"message\":\"Trace[1860435126]: ---\\\"Writing http response done\\\" 18ms (10:13:03.923)\",\"docker\":{\"container_id\":\"132c05cbd000604df3872742ad8ffc45150e161152a89cd3187c682e1d324ec1\"},\"kubernetes\":{\"container_name\":\"kube-apiserver\",\"namespace_name\":\"openshift-kube-apiserver\",\"pod_name\":\"kube-apiserver-master-1\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bcafbca3b814c6a005a964c00a5c164fa2b588e6365b48e3d11ca1380d30075\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bcafbca3b814c6a005a964c00a5c164fa2b588e6365b48e3d11ca1380d30075\",\"pod_id\":\"787ce140-bbae-4d9f-8319-1ab33290a9ce\",\"pod_ip\":\"10.20.184.206\",\"host\":\"master-1\",\"labels\":{\"apiserver\":\"true\",\"app\":\"openshift-kube-apiserver\",\"revision\":\"13\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"11333699-8997-45d0-aa84-db028e3c25d2\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-kube-apiserver\",\"olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"openshift_io_run-level\":\"0\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"flat_labels\":[\"apiserver=true\",\"app=openshift-kube-apiserver\",\"revision=13\"]},\"level\":\"unknown\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.206\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:04.222265+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":14132,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-infrastructure\"}},\"viaq_msg_id\":\"MjE4YWIwN2ItNzg3NC00MzgwLTkxMWYtNjFiMjAxZDMyMjI4\",\"log_type\":\"infrastructure\"}"},{"ts":"2025-04-01T10:13:03.924131846Z","line":"{\"@timestamp\":\"2025-04-01T10:13:03.924131846+00:00\",\"message\":\"Trace[1860435126]: ---\\\"Encode succeeded\\\" len:136323 98ms (10:13:03.844)]\",\"docker\":{\"container_id\":\"132c05cbd000604df3872742ad8ffc45150e161152a89cd3187c682e1d324ec1\"},\"kubernetes\":{\"container_name\":\"kube-apiserver\",\"namespace_name\":\"openshift-kube-apiserver\",\"pod_name\":\"kube-apiserver-master-1\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bcafbca3b814c6a005a964c00a5c164fa2b588e6365b48e3d11ca1380d30075\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bcafbca3b814c6a005a964c00a5c164fa2b588e6365b48e3d11ca1380d30075\",\"pod_id\":\"787ce140-bbae-4d9f-8319-1ab33290a9ce\",\"pod_ip\":\"10.20.184.206\",\"host\":\"master-1\",\"labels\":{\"apiserver\":\"true\",\"app\":\"openshift-kube-apiserver\",\"revision\":\"13\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"11333699-8997-45d0-aa84-db028e3c25d2\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-kube-apiserver\",\"olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"openshift_io_run-level\":\"0\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"flat_labels\":[\"apiserver=true\",\"app=openshift-kube-apiserver\",\"revision=13\"]},\"level\":\"unknown\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.206\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:04.222056+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":14131,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-infrastructure\"}},\"viaq_msg_id\":\"MDYzYjYxMWQtOTE3Yi00Y2RhLWIzMTYtZjc2YjkxYmIzZmMy\",\"log_type\":\"infrastructure\"}"},{"ts":"2025-04-01T10:13:03.924131846Z","line":"{\"@timestamp\":\"2025-04-01T10:13:03.924131846+00:00\",\"message\":\"Trace[1860435126]: ---\\\"About to Encode\\\" 485ms (10:13:03.746)\",\"docker\":{\"container_id\":\"132c05cbd000604df3872742ad8ffc45150e161152a89cd3187c682e1d324ec1\"},\"kubernetes\":{\"container_name\":\"kube-apiserver\",\"namespace_name\":\"openshift-kube-apiserver\",\"pod_name\":\"kube-apiserver-master-1\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bcafbca3b814c6a005a964c00a5c164fa2b588e6365b48e3d11ca1380d30075\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bcafbca3b814c6a005a964c00a5c164fa2b588e6365b48e3d11ca1380d30075\",\"pod_id\":\"787ce140-bbae-4d9f-8319-1ab33290a9ce\",\"pod_ip\":\"10.20.184.206\",\"host\":\"master-1\",\"labels\":{\"apiserver\":\"true\",\"app\":\"openshift-kube-apiserver\",\"revision\":\"13\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"11333699-8997-45d0-aa84-db028e3c25d2\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-kube-apiserver\",\"olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"openshift_io_run-level\":\"0\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"flat_labels\":[\"apiserver=true\",\"app=openshift-kube-apiserver\",\"revision=13\"]},\"level\":\"unknown\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.206\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:04.221742+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":14130,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-infrastructure\"}},\"viaq_msg_id\":\"MTk0OTI3N2QtMmE4NC00OGI1LTg3YjYtNzA1NWM1YzAxYjIw\",\"log_type\":\"infrastructure\"}"},{"ts":"2025-04-01T10:13:03.924131846Z","line":"{\"@timestamp\":\"2025-04-01T10:13:03.924131846+00:00\",\"message\":\"Trace[1860435126]: [\\\"GuaranteedUpdate etcd3\\\" audit-id:c9905265-66f2-4c8d-adc6-815669d29f7a,key:/apiserver.openshift.io/apirequestcounts/subjectaccessreviews.v1.authorization.k8s.io,type:*unstructured.Unstructured,resource:apirequestcounts.apiserver.openshift.io 699ms (10:13:03.224)\",\"docker\":{\"container_id\":\"132c05cbd000604df3872742ad8ffc45150e161152a89cd3187c682e1d324ec1\"},\"kubernetes\":{\"container_name\":\"kube-apiserver\",\"namespace_name\":\"openshift-kube-apiserver\",\"pod_name\":\"kube-apiserver-master-1\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bcafbca3b814c6a005a964c00a5c164fa2b588e6365b48e3d11ca1380d30075\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bcafbca3b814c6a005a964c00a5c164fa2b588e6365b48e3d11ca1380d30075\",\"pod_id\":\"787ce140-bbae-4d9f-8319-1ab33290a9ce\",\"pod_ip\":\"10.20.184.206\",\"host\":\"master-1\",\"labels\":{\"apiserver\":\"true\",\"app\":\"openshift-kube-apiserver\",\"revision\":\"13\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"11333699-8997-45d0-aa84-db028e3c25d2\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-kube-apiserver\",\"olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"openshift_io_run-level\":\"0\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"flat_labels\":[\"apiserver=true\",\"app=openshift-kube-apiserver\",\"revision=13\"]},\"level\":\"unknown\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.206\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:04.221483+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":14129,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-infrastructure\"}},\"viaq_msg_id\":\"YTdlZmJmMTgtYjlmNS00YTcyLWFjZTYtZjY2Nzg5OWRhZmJl\",\"log_type\":\"infrastructure\"}"},{"ts":"2025-04-01T10:13:03.924131846Z","line":"{\"@timestamp\":\"2025-04-01T10:13:03.924131846+00:00\",\"message\":\"Trace[1860435126]: ---\\\"Conversion done\\\" 80ms (10:13:03.217)\",\"docker\":{\"container_id\":\"132c05cbd000604df3872742ad8ffc45150e161152a89cd3187c682e1d324ec1\"},\"kubernetes\":{\"container_name\":\"kube-apiserver\",\"namespace_name\":\"openshift-kube-apiserver\",\"pod_name\":\"kube-apiserver-master-1\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bcafbca3b814c6a005a964c00a5c164fa2b588e6365b48e3d11ca1380d30075\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bcafbca3b814c6a005a964c00a5c164fa2b588e6365b48e3d11ca1380d30075\",\"pod_id\":\"787ce140-bbae-4d9f-8319-1ab33290a9ce\",\"pod_ip\":\"10.20.184.206\",\"host\":\"master-1\",\"labels\":{\"apiserver\":\"true\",\"app\":\"openshift-kube-apiserver\",\"revision\":\"13\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"11333699-8997-45d0-aa84-db028e3c25d2\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-kube-apiserver\",\"olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"openshift_io_run-level\":\"0\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"flat_labels\":[\"apiserver=true\",\"app=openshift-kube-apiserver\",\"revision=13\"]},\"level\":\"unknown\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.206\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:04.221183+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":14128,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-infrastructure\"}},\"viaq_msg_id\":\"NTJiNzI5M2UtM2ZmZS00ODRkLWFiNzctYzQ4NTE0NGZlZDc1\",\"log_type\":\"infrastructure\"}"},{"ts":"2025-04-01T10:13:03.924131846Z","line":"{\"@timestamp\":\"2025-04-01T10:13:03.924131846+00:00\",\"message\":\"I0401 10:13:03.923852 19 trace.go:236] Trace[1860435126]: \\\"Update\\\" accept:application/json,audit-id:c9905265-66f2-4c8d-adc6-815669d29f7a,client:::1,protocol:HTTP/2.0,resource:apirequestcounts,scope:resource,url:/apis/apiserver.openshift.io/v1/apirequestcounts/subjectaccessreviews.v1.authorization.k8s.io/status,user-agent:kube-apiserver/v1.27.16+03a907c (linux/ppc64le) kubernetes/994a6d8,verb:PUT (01-Apr-2025 10:13:03.130) (total time: 793ms):\",\"docker\":{\"container_id\":\"132c05cbd000604df3872742ad8ffc45150e161152a89cd3187c682e1d324ec1\"},\"kubernetes\":{\"container_name\":\"kube-apiserver\",\"namespace_name\":\"openshift-kube-apiserver\",\"pod_name\":\"kube-apiserver-master-1\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bcafbca3b814c6a005a964c00a5c164fa2b588e6365b48e3d11ca1380d30075\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8bcafbca3b814c6a005a964c00a5c164fa2b588e6365b48e3d11ca1380d30075\",\"pod_id\":\"787ce140-bbae-4d9f-8319-1ab33290a9ce\",\"pod_ip\":\"10.20.184.206\",\"host\":\"master-1\",\"labels\":{\"apiserver\":\"true\",\"app\":\"openshift-kube-apiserver\",\"revision\":\"13\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"11333699-8997-45d0-aa84-db028e3c25d2\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-kube-apiserver\",\"olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f\":\"\",\"openshift_io_cluster-monitoring\":\"true\",\"openshift_io_run-level\":\"0\",\"pod-security_kubernetes_io_audit\":\"privileged\",\"pod-security_kubernetes_io_enforce\":\"privileged\",\"pod-security_kubernetes_io_warn\":\"privileged\"},\"flat_labels\":[\"apiserver=true\",\"app=openshift-kube-apiserver\",\"revision=13\"]},\"level\":\"info\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.206\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:04.220890+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":14127,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-infrastructure\"}},\"viaq_msg_id\":\"YWJmNzM0NmYtMmQzNS00ZmYxLTgzYmEtNGExMTI2ZTdmZWUz\",\"log_type\":\"infrastructure\"}"}]},{"labels":"{fluentd_thread=\"flush_thread_0\", kubernetes_container_name=\"conversion-webhook-server\", kubernetes_host=\"master-1\", kubernetes_namespace_name=\"openshift-console-operator\", kubernetes_pod_name=\"console-operator-d4c9cc888-x7x2b\", log_type=\"infrastructure\"}","entries":[{"ts":"2025-04-01T10:13:14.258656284Z","line":"{\"@timestamp\":\"2025-04-01T10:13:14.258656284+00:00\",\"message\":\"I0401 10:13:14.258623 1 framework.go:107] sending response: \u0026ConversionResponse{UID:ffeeb00b-2810-4489-b3b2-2273824b020f,ConvertedObjects:[]RawExtension{{[] 0xc0000121c0},},Result:{{ } { \u003cnil\u003e} Success nil 0},}\",\"docker\":{\"container_id\":\"59003cb1e205b8b3596fe8f45871d466a9f5c90c67e7d2fd416ab0c201556f90\"},\"kubernetes\":{\"container_name\":\"conversion-webhook-server\",\"namespace_name\":\"openshift-console-operator\",\"pod_name\":\"console-operator-d4c9cc888-x7x2b\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4c807bb2b201e981dd409d5dcfbcd775918d2382867d81996a03ae1ec9611c99\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4c807bb2b201e981dd409d5dcfbcd775918d2382867d81996a03ae1ec9611c99\",\"pod_id\":\"50fd5b89-7e4e-4b72-8826-e936df786e31\",\"pod_ip\":\"10.130.1.228\",\"host\":\"master-1\",\"labels\":{\"name\":\"console-operator\",\"pod-template-hash\":\"d4c9cc888\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"f31af66a-3210-4b29-99f0-ca40f4866e76\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-console-operator\",\"olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f\":\"\",\"openshift_io_cluster-monitoring\":\"true\"},\"flat_labels\":[\"name=console-operator\",\"pod-template-hash=d4c9cc888\"]},\"level\":\"info\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.206\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:14.273249+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":15108,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-infrastructure\"}},\"viaq_msg_id\":\"NDM1ZTUxNTktN2RkYi00ZTI3LTg4NDgtMTQ3MzhmZWExN2Ew\",\"log_type\":\"infrastructure\"}"},{"ts":"2025-04-01T10:13:14.258269146Z","line":"{\"@timestamp\":\"2025-04-01T10:13:14.258269146+00:00\",\"message\":\"I0401 10:13:14.258234 1 converter.go:27] converting \\\"monitoring-plugin\\\" object from 'console.openshift.io/v1alpha1' into 'console.openshift.io/v1'\",\"docker\":{\"container_id\":\"59003cb1e205b8b3596fe8f45871d466a9f5c90c67e7d2fd416ab0c201556f90\"},\"kubernetes\":{\"container_name\":\"conversion-webhook-server\",\"namespace_name\":\"openshift-console-operator\",\"pod_name\":\"console-operator-d4c9cc888-x7x2b\",\"container_image\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4c807bb2b201e981dd409d5dcfbcd775918d2382867d81996a03ae1ec9611c99\",\"container_image_id\":\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4c807bb2b201e981dd409d5dcfbcd775918d2382867d81996a03ae1ec9611c99\",\"pod_id\":\"50fd5b89-7e4e-4b72-8826-e936df786e31\",\"pod_ip\":\"10.130.1.228\",\"host\":\"master-1\",\"labels\":{\"name\":\"console-operator\",\"pod-template-hash\":\"d4c9cc888\"},\"master_url\":\"https://kubernetes.default.svc\",\"namespace_id\":\"f31af66a-3210-4b29-99f0-ca40f4866e76\",\"namespace_labels\":{\"kubernetes_io_metadata_name\":\"openshift-console-operator\",\"olm_operatorgroup_uid_8840daad-9ed2-4597-ae63-1ebed57d5a3f\":\"\",\"openshift_io_cluster-monitoring\":\"true\"},\"flat_labels\":[\"name=console-operator\",\"pod-template-hash=d4c9cc888\"]},\"level\":\"info\",\"hostname\":\"master-1\",\"pipeline_metadata\":{\"collector\":{\"ipaddr4\":\"10.20.184.206\",\"inputname\":\"fluent-plugin-systemd\",\"name\":\"fluentd\",\"received_at\":\"2025-04-01T10:13:14.273039+00:00\",\"version\":\"1.16.2 1.6.0\"}},\"openshift\":{\"sequence\":15107,\"cluster_id\":\"657154e1-ef9c-426e-8079-d58eaee0a17b\",\"labels\":{\"logs\":\"5.8_414-infrastructure\"}},\"viaq_msg_id\":\"ODQ2MjgzODItN2MwNC00ZjY1LThiYTYtMzRjNzZjN2VkODI5\",\"log_type\":\"infrastructure\"}"} ```