Loading...

Type: Bug
Resolution: Done
Priority: Normal
Fix Version/s: Logging 5.6.0
Affects Version/s: None
Component/s: Log Collection
Labels:

Blocked:
False
Blocked Reason:
None
Ready:
False
QE Status:
VERIFIED
Release Note Text:
Before this update, the `.level` and`.structure.level` fields in Fluentd could contain different values. With this update, the values are the same for each field.

Sprint:
Log Collection - Sprint 224, Log Collection - Sprint 225, Log Collection - Sprint 226, Log Collection - Sprint 227
Severity:
Low

SFDC Cases Links:
SFDC Cases Counter:
SFDC Cases Open:

One of my customers is facing the issue randomly in the JSON logs the `.level` field they are getting the "ERROR" but in the `.structure.level` field they are getting "INFO"

Sample log below:

{  "_index": "app-dc-prod-003442",  "_type": "_doc",  "_id": "N2JmODA5ZmQtYjU2MS00NDM0LWFlMWMtMTA3MzkyMGNlNzAx",  "_score": 1,  "_source": {    "kubernetes": {      "container_image_id": "nexus-prod-xx.xx.xx/mo/dc/customer-applications@sha256:a4fe0828330aa489eccdc70eb143debf94577bcde4450f2a95ebf9bebb628f56",      "container_name": "customer-applications",      "namespace_id": "cf0dbd04-56e7-4ad1-9647-7a7a711b8208",      "flat_labels": [        "app=customer-applications",        "deployment=customer-applications-10",        "deploymentconfig=customer-applications",        "subDomain=application"      ],      "pod_ip": "10.xx.xx.xx",      "host": "ip-xx.xx.xx.eu-west-1.compute.internal",      "master_url": "https://kubernetes.default.svc",      "pod_id": "d3538ec9-7802-471a-9137-5d8132e918b3",      "namespace_labels": {        "owned-by": "tenant-config-operator",        "kubernetes_io/metadata_name": "dc-prod",        "argocd_argoproj_io/managed-by": "tooling-dc",        "workload": "apps",        "tenant": "dc"      },      "container_image": "nexus-prod-xx.xx.xx/mo/dc/customer-applications:1.0.0-681-e110655b",      "namespace_name": "dc-prod",      "pod_name": "customer-applications-10-r6vv7"    },    "viaq_msg_id": "N2JmODA5ZmQtYjU2MS00NDM0LWFlMWMtMTA3MzkyMGNlNzAx",    "level": "error",    "openshift": {      "sequence": 7770876    },    "message": "{\"@timestamp\":\"2022-07-11T14:05:44.995+01:00\",\"message\":\"Unprocessable entity\",\"logger_name\":\"uk.co.mo.services.ola.exceptions.ErrorExceptionResolver\",\"thread_name\":\"http-nio-10000-exec-5\",\"level\":\"INFO\",\"stack_trace\":\"uk.co.mo.services.ola.exceptions.UnprocessableEntityException: null\\n\\tat uk.co.mo.services.ola.preapplication.domain.service.FindApplicationsQuery.find(FindApplicationsQuery.java:60)\\n\\tat uk.co.mo.services.ola.preapplication.domain.service.PreApplicationService.findApplications(PreApplicationService.java:88)\\n\\tat uk.co.mo.services.ola.preapplication.web.PreApplicationController.search(PreApplicationController.java:52)\\n\\tat jdk.internal.reflect.GeneratedMethodAccessor396.invoke(Unknown Source)\\n\\tat java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\\n\\tat java.base/java.lang.reflect.Method.invoke(Method.java:566)\\n\\tat org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)\\n\\tat org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)\\n\\tat org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)\\n\\tat org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)\\n\\tat org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)\\n\\tat org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)\\n\\tat org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1067)\\n\\tat org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:963)\\n\\tat org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)\\n\\tat org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:898)\\n\\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:655)\\n\\tat org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)\\n\\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:764)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:227)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\\n\\tat org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\\n\\tat uk.co.mo.services.ola.logging.RequestLoggingFilter.logRequestAndResponse(RequestLoggingFilter.java:66)\\n\\tat uk.co.mo.services.ola.logging.RequestLoggingFilter.doFilter(RequestLoggingFilter.java:54)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\\n\\tat uk.co.mo.services.ola.logging.CorrelationInfoLoggingFilter.doFilter(CorrelationInfoLoggingFilter.java:44)\\n\\tat uk.co.mo.services.ola.logging.CorrelationInfoLoggingFilter.doFilter(CorrelationInfoLoggingFilter.java:36)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\\n\\tat org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)\\n\\tat org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\\n\\tat org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93)\\n\\tat org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\\n\\tat org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)\\n\\tat org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\\n\\tat org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)\\n\\tat org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\\n\\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\\n\\tat org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:197)\\n\\tat org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97)\\n\\tat org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541)\\n\\tat org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:135)\\n\\tat org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)\\n\\tat org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78)\\n\\tat org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:769)\\n\\tat org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:360)\\n\\tat org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:399)\\n\\tat org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)\\n\\tat org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:890)\\n\\tat org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1743)\\n\\tat org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)\\n\\tat org.apache.tomcat.util.threads.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1191)\\n\\tat org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:659)\\n\\tat org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)\\n\\tat java.base/java.lang.Thread.run(Thread.java:829)\\n\",\"X-MO-Dealer-Code\":\"\",\"X-MO-User-ID\":\"78e518f9-0076-40be-91cf-d92db978d77b\",\"X-MO-Correlation-ID\":\"4b4d1dbe-e199-4799-aa53-03aa9921b09c\"}",    "docker": {      "container_id": "46246900cc2fddde03f317d147cafafdd7abcf929b19a939156c42f7bf8935af"    },    "hostname": "ip-xx.xx.xx.eu-west-1.compute.internal",    "log_type": "application",    "@timestamp": "2022-07-11T13:05:44.996127+00:00",    "pipeline_metadata": {      "collector": {        "received_at": "2022-07-11T13:05:45.008416+00:00",        "name": "fluentd",        "inputname": "fluent-plugin-systemd",        "version": "1.14.5 1.6.0",        "ipaddr4": "10.xx.xx.xx"      }    },    "structured": {      "@timestamp": "2022-07-11T14:05:44.995+01:00",      "level": "INFO",      "thread_name": "http-nio-10000-exec-5",      "X-MO-User-ID": "78e518f9-0076-40be-91cf-d92db978d77b",      "X-MO-Correlation-ID": "4b4d1dbe-e199-4799-aa53-03aa9921b09c",      "logger_name": "uk.co.mo.services.ola.exceptions.ErrorExceptionResolver",      "stack_trace": "uk.co.mo.services.ola.exceptions.UnprocessableEntityException: null\n\tat uk.co.mo.services.ola.preapplication.domain.service.FindApplicationsQuery.find(FindApplicationsQuery.java:60)\n\tat uk.co.mo.services.ola.preapplication.domain.service.PreApplicationService.findApplications(PreApplicationService.java:88)\n\tat uk.co.mo.services.ola.preapplication.web.PreApplicationController.search(PreApplicationController.java:52)\n\tat jdk.internal.reflect.GeneratedMethodAccessor396.invoke(Unknown Source)\n\tat java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.base/java.lang.reflect.Method.invoke(Method.java:566)\n\tat org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)\n\tat org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)\n\tat org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)\n\tat org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)\n\tat org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)\n\tat org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)\n\tat org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1067)\n\tat org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:963)\n\tat org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)\n\tat org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:898)\n\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:655)\n\tat org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)\n\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:764)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:227)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\n\tat org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\n\tat uk.co.mo.services.ola.logging.RequestLoggingFilter.logRequestAndResponse(RequestLoggingFilter.java:66)\n\tat uk.co.mo.services.ola.logging.RequestLoggingFilter.doFilter(RequestLoggingFilter.java:54)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\n\tat uk.co.mo.services.ola.logging.CorrelationInfoLoggingFilter.doFilter(CorrelationInfoLoggingFilter.java:44)\n\tat uk.co.mo.services.ola.logging.CorrelationInfoLoggingFilter.doFilter(CorrelationInfoLoggingFilter.java:36)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\n\tat org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)\n\tat org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\n\tat org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93)\n\tat org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\n\tat org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)\n\tat org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\n\tat org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)\n\tat org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)\n\tat org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:197)\n\tat org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97)\n\tat org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541)\n\tat org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:135)\n\tat org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)\n\tat org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78)\n\tat org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:769)\n\tat org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:360)\n\tat org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:399)\n\tat org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)\n\tat org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:890)\n\tat org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1743)\n\tat org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)\n\tat org.apache.tomcat.util.threads.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1191)\n\tat org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:659)\n\tat org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\n",      "message": "Unprocessable entity"    }  },  "fields": {    "@timestamp": [      "2022-07-11T13:05:44.996Z"    ],    "pipeline_metadata.collector.received_at": [      "2022-07-11T13:05:45.008Z"    ]  }}