Uploaded image for project: 'OpenShift Logging'
  1. OpenShift Logging
  2. LOG-3358

JSON logs without outputDefaults field are not parser

XMLWordPrintable

    • False
    • None
    • False
    • Logging
    • NEW
    • NEW
    • Medium
    • Hide

      1) Create a CLF instance without adding "outputDefaults" fields

      spec:
        pipelines:
        - inputRefs:
          - application
          outputRefs:
          - default
          parse: json

      2) Deploy an application and send JSON logs
      $ oc new-app rails-postgresql-example

      $ oc rsh rails-postgresql-example-1-7wcnz
      sh-4.4$ echo '{"level":"info","name":"fred","home":"bedrock"}' >> /proc/1/fd/1
      sh-4.4$ echo '{"level":"info","name":"fred","home":"bedrock"}' >> /proc/1/fd/1
      sh-4.4$ echo '{"level":"info","name":"fred","home":"bedrock"}' >> /proc/1/fd/1
      sh-4.4$ echo '{"level":"info","name":"fred","home":"bedrock"}' >> /proc/1/fd/1

      3) Check in Kibana that the log is not parsed:

      {
        "_index": "app-000001",
        "_type": "_doc",
        "_id": "MDc4ZmQ4OTYtNmUxYi00ZGMxLTkwNDctNWUwZmE5YzA5OTVl",
        "_version": 1,
        "_score": null,
        "_source": {
          "kubernetes": {
            "container_image_id": "image-registry.openshift-image-registry.svc:5000/adritest/rails-postgresql-example@sha256:e87ed7ea43dbf33b3b54fec51f189511df92cf701a928360319a5089009ceb09",
            "container_name": "rails-postgresql-example",
            "namespace_id": "13ad3465-9a76-43a4-9c1c-468896a44c88",
            "flat_labels": [
              "deployment=rails-postgresql-example-1",
              "deploymentconfig=rails-postgresql-example",
              "name=rails-postgresql-example"
            ],
            "pod_ip": "10.128.2.15",
            "host": "worker-2.adritest.lab.psi.pnq2.redhat.com",
            "master_url": "https://kubernetes.default.svc",
            "pod_id": "aea407ab-5351-4d65-9653-4e21a6cebee2",
            "namespace_labels": {
              "pod-security.kubernetes.io/warn-version": "v1.24",
              "pod-security.kubernetes.io/audit-version": "v1.24",
              "pod-security.kubernetes.io/audit": "restricted",
              "pod-security.kubernetes.io/warn": "restricted",
              "kubernetes.io/metadata.name": "adritest"
            },
            "container_image": "image-registry.openshift-image-registry.svc:5000/adritest/rails-postgresql-example@sha256:e87ed7ea43dbf33b3b54fec51f189511df92cf701a928360319a5089009ceb09",
            "namespace_name": "adritest",
            "pod_name": "rails-postgresql-example-1-7wcnz"
          },
          "viaq_msg_id": "MDc4ZmQ4OTYtNmUxYi00ZGMxLTkwNDctNWUwZmE5YzA5OTVl",
          "level": "info",
          "openshift": {
            "sequence": 97
          },
          "message": "{\"level\":\"info\",\"name\":\"fred\",\"home\":\"bedrock\"}",
          "docker": {
            "container_id": "1b54c92a8302f6789ac8fcc23d4d8f0b87f17890d5cb63a24172ac02bf3fb8e8"
          },
          "hostname": "worker-2.adritest.lab.psi.pnq2.redhat.com",
          "log_type": "application",
          "@timestamp": "2022-12-01T12:21:44.368172331+00:00",
          "pipeline_metadata": {
            "collector": {
              "received_at": "2022-12-01T12:21:44.368844+00:00",
              "name": "fluentd",
              "inputname": "fluent-plugin-systemd",
              "version": "1.14.6 1.6.0",
              "ipaddr4": "10.74.210.205"
            }
          }
        },
        "fields": {
          "@timestamp": [
            "2022-12-01T12:21:44.368Z"
          ],
          "pipeline_metadata.collector.received_at": [
            "2022-12-01T12:21:44.368Z"
          ]
        },
        "sort": [
          1669897304368
        ]
      }

       

      Additionally, modifying the CLF instance:

      spec:
       inputs:
       - application:
           namespaces:
           - adritest
         name: my-parsed-app-logs
       outputDefaults:
         elasticsearch:
           structuredTypeKey: openshift.labels.myLabel
       pipelines:
       - inputRefs:
         - application
         labels:
           myLabel: ParsedApp
         outputRefs:
         - default
         parse: json
      
      

      And repeating steps 2 and 3, we can observe that the logs are parsed properly in Kibana (check the structured message ).

      {
        "_index": "app-parsedapp-000001",
        "_type": "_doc",
        "_id": "NTJlODZkMDgtNDA2NC00ODM3LTlmZDMtMGE1NDIwOTc1MDU5",
        "_version": 1,
        "_score": null,
        "_source": {
          "kubernetes": {
            "container_image_id": "image-registry.openshift-image-registry.svc:5000/adritest/rails-postgresql-example@sha256:e87ed7ea43dbf33b3b54fec51f189511df92cf701a928360319a5089009ceb09",
            "container_name": "rails-postgresql-example",
            "namespace_id": "13ad3465-9a76-43a4-9c1c-468896a44c88",
            "flat_labels": [
              "deployment=rails-postgresql-example-1",
              "deploymentconfig=rails-postgresql-example",
              "name=rails-postgresql-example"
            ],
            "pod_ip": "10.128.2.15",
            "host": "worker-2.adritest.lab.psi.pnq2.redhat.com",
            "master_url": "https://kubernetes.default.svc",
            "pod_id": "aea407ab-5351-4d65-9653-4e21a6cebee2",
            "namespace_labels": {
              "pod-security.kubernetes.io/warn-version": "v1.24",
              "pod-security.kubernetes.io/audit-version": "v1.24",
              "pod-security.kubernetes.io/audit": "restricted",
              "pod-security.kubernetes.io/warn": "restricted",
              "kubernetes.io/metadata.name": "adritest"
            },
            "container_image": "image-registry.openshift-image-registry.svc:5000/adritest/rails-postgresql-example@sha256:e87ed7ea43dbf33b3b54fec51f189511df92cf701a928360319a5089009ceb09",
            "namespace_name": "adritest",
            "pod_name": "rails-postgresql-example-1-7wcnz"
          },
          "viaq_msg_id": "NTJlODZkMDgtNDA2NC00ODM3LTlmZDMtMGE1NDIwOTc1MDU5",
          "level": "info",
          "openshift": {
            "labels": {
              "myLabel": "ParsedApp"
            }
          },
          "docker": {
            "container_id": "1b54c92a8302f6789ac8fcc23d4d8f0b87f17890d5cb63a24172ac02bf3fb8e8"
          },
          "hostname": "worker-2.adritest.lab.psi.pnq2.redhat.com",
          "log_type": "application",
          "@timestamp": "2022-12-01T12:14:41.722003634+00:00",
          "pipeline_metadata": {
            "collector": {
              "received_at": "2022-12-01T12:14:41.722891+00:00",
              "name": "fluentd",
              "inputname": "fluent-plugin-systemd",
              "version": "1.14.6 1.6.0",
              "ipaddr4": "10.74.210.205"
            }
          },
          "structured": {
            "level": "info",
            "name": "fred",
            "home": "bedrock"
          }
        },
        "fields": {
          "@timestamp": [
            "2022-12-01T12:14:41.722Z"
          ],
          "pipeline_metadata.collector.received_at": [
            "2022-12-01T12:14:41.722Z"
          ]
        },
        "sort": [
          1669896881722
        ]
      }

       

       

       

       

       

       

       

       

       

       

       

       

      Show
      1) Create a CLF instance without adding "outputDefaults" fields spec:   pipelines:   - inputRefs:     - application     outputRefs:     - default     parse: json 2) Deploy an application and send JSON logs $ oc new-app rails-postgresql-example $ oc rsh rails-postgresql-example-1-7wcnz sh-4.4$ echo '{ "level" : "info" , "name" : "fred" , "home" : "bedrock" }' >> /proc/1/fd/1 sh-4.4$ echo '{ "level" : "info" , "name" : "fred" , "home" : "bedrock" }' >> /proc/1/fd/1 sh-4.4$ echo '{ "level" : "info" , "name" : "fred" , "home" : "bedrock" }' >> /proc/1/fd/1 sh-4.4$ echo '{ "level" : "info" , "name" : "fred" , "home" : "bedrock" }' >> /proc/1/fd/1 3) Check in Kibana that the log is not parsed: {   "_index" : "app-000001" ,   "_type" : "_doc" ,   "_id" : "MDc4ZmQ4OTYtNmUxYi00ZGMxLTkwNDctNWUwZmE5YzA5OTVl" ,   "_version" : 1,   "_score" : null ,   "_source" : {     "kubernetes" : {       "container_image_id" : "image-registry.openshift-image-registry.svc:5000/adritest/rails-postgresql-example@sha256:e87ed7ea43dbf33b3b54fec51f189511df92cf701a928360319a5089009ceb09" ,       "container_name" : "rails-postgresql-example" ,       "namespace_id" : "13ad3465-9a76-43a4-9c1c-468896a44c88" ,       "flat_labels" : [         "deployment=rails-postgresql-example-1" ,         "deploymentconfig=rails-postgresql-example" ,         "name=rails-postgresql-example"       ],       "pod_ip" : "10.128.2.15" ,       "host" : "worker-2.adritest.lab.psi.pnq2.redhat.com" ,       "master_url" : "https: //kubernetes. default .svc" ,       "pod_id" : "aea407ab-5351-4d65-9653-4e21a6cebee2" ,       "namespace_labels" : {         "pod-security.kubernetes.io/warn-version" : "v1.24" ,         "pod-security.kubernetes.io/audit-version" : "v1.24" ,         "pod-security.kubernetes.io/audit" : "restricted" ,         "pod-security.kubernetes.io/warn" : "restricted" ,         "kubernetes.io/metadata.name" : "adritest"       },       "container_image" : "image-registry.openshift-image-registry.svc:5000/adritest/rails-postgresql-example@sha256:e87ed7ea43dbf33b3b54fec51f189511df92cf701a928360319a5089009ceb09" ,       "namespace_name" : "adritest" ,       "pod_name" : "rails-postgresql-example-1-7wcnz"     },     "viaq_msg_id" : "MDc4ZmQ4OTYtNmUxYi00ZGMxLTkwNDctNWUwZmE5YzA5OTVl" ,     "level" : "info" ,     "openshift" : {       "sequence" : 97     },     "message" : "{\" level\ ":\" info\ ",\" name\ ":\" fred\ ",\" home\ ":\" bedrock\ "}" ,     "docker" : {       "container_id" : "1b54c92a8302f6789ac8fcc23d4d8f0b87f17890d5cb63a24172ac02bf3fb8e8"     },     "hostname" : "worker-2.adritest.lab.psi.pnq2.redhat.com" ,     "log_type" : "application" ,     "@timestamp" : "2022-12-01T12:21:44.368172331+00:00" ,     "pipeline_metadata" : {       "collector" : {         "received_at" : "2022-12-01T12:21:44.368844+00:00" ,         "name" : "fluentd" ,         "inputname" : "fluent-plugin-systemd" ,         "version" : "1.14.6 1.6.0" ,         "ipaddr4" : "10.74.210.205"       }     }   },   "fields" : {     "@timestamp" : [       "2022-12-01T12:21:44.368Z"     ],     "pipeline_metadata.collector.received_at" : [       "2022-12-01T12:21:44.368Z"     ]   },   "sort" : [     1669897304368   ] }   Additionally, modifying the CLF instance: spec:  inputs:  - application:      namespaces:      - adritest    name: my-parsed-app-logs  outputDefaults:    elasticsearch:      structuredTypeKey: openshift.labels.myLabel  pipelines:  - inputRefs:    - application    labels:      myLabel: ParsedApp    outputRefs:    - default    parse: json And repeating steps 2 and 3, we can observe that the logs are parsed properly in Kibana (check the structured message ). {   "_index" : "app-parsedapp-000001" ,   "_type" : "_doc" ,   "_id" : "NTJlODZkMDgtNDA2NC00ODM3LTlmZDMtMGE1NDIwOTc1MDU5" ,   "_version" : 1,   "_score" : null ,   "_source" : {     "kubernetes" : {       "container_image_id" : "image-registry.openshift-image-registry.svc:5000/adritest/rails-postgresql-example@sha256:e87ed7ea43dbf33b3b54fec51f189511df92cf701a928360319a5089009ceb09" ,       "container_name" : "rails-postgresql-example" ,       "namespace_id" : "13ad3465-9a76-43a4-9c1c-468896a44c88" ,       "flat_labels" : [         "deployment=rails-postgresql-example-1" ,         "deploymentconfig=rails-postgresql-example" ,         "name=rails-postgresql-example"       ],       "pod_ip" : "10.128.2.15" ,       "host" : "worker-2.adritest.lab.psi.pnq2.redhat.com" ,       "master_url" : "https: //kubernetes. default .svc" ,       "pod_id" : "aea407ab-5351-4d65-9653-4e21a6cebee2" ,       "namespace_labels" : {         "pod-security.kubernetes.io/warn-version" : "v1.24" ,         "pod-security.kubernetes.io/audit-version" : "v1.24" ,         "pod-security.kubernetes.io/audit" : "restricted" ,         "pod-security.kubernetes.io/warn" : "restricted" ,         "kubernetes.io/metadata.name" : "adritest"       },       "container_image" : "image-registry.openshift-image-registry.svc:5000/adritest/rails-postgresql-example@sha256:e87ed7ea43dbf33b3b54fec51f189511df92cf701a928360319a5089009ceb09" ,       "namespace_name" : "adritest" ,       "pod_name" : "rails-postgresql-example-1-7wcnz"     },     "viaq_msg_id" : "NTJlODZkMDgtNDA2NC00ODM3LTlmZDMtMGE1NDIwOTc1MDU5" ,     "level" : "info" ,     "openshift" : {       "labels" : {         "myLabel" : "ParsedApp"       }     },     "docker" : {       "container_id" : "1b54c92a8302f6789ac8fcc23d4d8f0b87f17890d5cb63a24172ac02bf3fb8e8"     },     "hostname" : "worker-2.adritest.lab.psi.pnq2.redhat.com" ,     "log_type" : "application" ,     "@timestamp" : "2022-12-01T12:14:41.722003634+00:00" ,     "pipeline_metadata" : {       "collector" : {         "received_at" : "2022-12-01T12:14:41.722891+00:00" ,         "name" : "fluentd" ,         "inputname" : "fluent-plugin-systemd" ,         "version" : "1.14.6 1.6.0" ,         "ipaddr4" : "10.74.210.205"       }     },     "structured" : {       "level" : "info" ,       "name" : "fred" ,       "home" : "bedrock"     }   },   "fields" : {     "@timestamp" : [       "2022-12-01T12:14:41.722Z"     ],     "pipeline_metadata.collector.received_at" : [       "2022-12-01T12:14:41.722Z"     ]   },   "sort" : [     1669896881722   ] }                        

       

      OCP Version 4.11.13

      RHOL Version 5.5.4

       

       

      JSON logs are only parsed when adding "outputDefaults" fields in the ClusterLogForwarder (CLF) instance.

      "structuredTypeKey" and "structuredTypeName" parameters are optional and only important for elasticsearch but not for the different third-party systems.

      Without the previous parameter, new custom app indices are not created (correct) but JSON logs are not parsed.

      In addition, in the documentation, we can find some examples with the same CLF instance structure (without adding "outputDefaults" parameter)-->https://docs.openshift.com/container-platform/4.11/logging/cluster-logging-external.html

       

       

       

       

       

       

            Unassigned Unassigned
            acandelp Adrian Candel
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

              Created:
              Updated:
              Resolved: