Uploaded image for project: 'OpenShift Logging'
  1. OpenShift Logging
  2. LOG-3195

[Vector] logs parsed into structured when json is set without structured types.

XMLWordPrintable

    • False
    • None
    • False
    • NEW
    • VERIFIED
    • Hide
      Before this update, Vector parsed the message field when JSON parsing was enabled without also defining structuredTypeKey or structuredTypeName values. With this update, a value is required for either structuredTypeKey or structuredTypeName when writing structured logs to Elasticsearch.
      Show
      Before this update, Vector parsed the message field when JSON parsing was enabled without also defining structuredTypeKey or structuredTypeName values. With this update, a value is required for either structuredTypeKey or structuredTypeName when writing structured logs to Elasticsearch.

      Description of problem:

      logs parsed into structured when json is set without structured types.

      Version-Release number of selected component (if applicable):

      Version of components:

      Cluster-logging.5.5.2

      Elasticsearch-operator.5.5.2

      Server Version: 4.11.0-0.nightly-2022-10-17-040259

      How reproducible:

      Always

      Steps to Reproduce:

      *Deploy a log generator pod which generates json logs.

      oc new-project test
      oc new-app -f https://gitlab.cee.redhat.com/aosqe/aosqe-tools/-/raw/master/logging/log_gen/container_json_log_template.json
      {"message": "MERGE_JSON_LOG=true", "level": "debug","Layer1": "layer1 0", "layer2": {"name":"Layer2 1", "tips":"Decide by PRESERVE_JSON_LOG"}, "StringNumber":"10", "Number": 10,"foo.bar":"Dot Item","{foobar}":"Brace Item","[foobar]":"Bracket Item", "foo:bar":"Colon Item","foo bar":"Space Item" }
      

      *Create CLF to forward logs to default log store with parse: json

      apiVersion: logging.openshift.io/v1
      kind: ClusterLogForwarder
      metadata:
        name: instance
        namespace: openshift-logging
      spec:
        pipelines:
        - inputRefs:
          - audit
          - infrastructure
          name: forward-infra-audit
          outputRefs:
          - default
        - inputRefs:
          - application
          name: forward-app
          outputRefs:
          - default
          parse: json
      

      *Create Cluster Logging instance with Fluentd as collector and check the application logs in the log store.

      apiVersion: "logging.openshift.io/v1"
      kind: "ClusterLogging"
      metadata:
        name: "instance" 
        namespace: "openshift-logging"
      spec:
        managementState: "Managed"  
        logStore:
          type: "elasticsearch"  
          retentionPolicy: 
            application:
              maxAge: 10h
            infra:
              maxAge: 10h
            audit:
              maxAge: 10h
          elasticsearch:
            nodeCount: 1 
            storage: {} 
            resources: 
                limits:
                  memory: "4Gi"
                requests:
                  memory: "1Gi"
            proxy: 
              resources:
                limits:
                  memory: 256Mi
                requests:
                  memory: 256Mi
            redundancyPolicy: "ZeroRedundancy"
        visualization:
          type: "kibana"  
          kibana:
            replicas: 1
        collection:
          logs:
            type: "fluentd"  
            fluentd: {}

      *Check log records in Elasticsearch with Fluentd as collector.

      {
        "_index": "app-000001",
        "_type": "_doc",
        "_id": "YzdkNTExZDctMzUxMS00MmY0LWJjY2MtZWM5N2MwMTFkN2Mx",
        "_version": 1,
        "_score": null,
        "_source": {
          "kubernetes": {
            "container_image_id": "quay.io/openshifttest/ocp-logtest@sha256:16232868ba1143721b786dbabb3f7384645acb663fadb4af48e9ea1228a67635",
            "container_name": "loggen-qa-json",
            "namespace_id": "db10b717-ee16-43eb-afef-3c935e5e7ac4",
            "pod_ip": "10.131.0.69",
            "host": "ip-10-0-148-10.us-east-2.compute.internal",
            "master_url": "https://kubernetes.default.svc",
            "pod_id": "aa174984-2070-4a02-b0c4-f905bc4f714b",
            "namespace_labels": {
              "pod-security.kubernetes.io/warn-version": "v1.24",
              "pod-security.kubernetes.io/audit-version": "v1.24",
              "pod-security.kubernetes.io/audit": "restricted",
              "pod-security.kubernetes.io/warn": "restricted",
              "kubernetes.io/metadata.name": "test"
            },
            "container_image": "quay.io/openshifttest/ocp-logtest@sha256:16232868ba1143721b786dbabb3f7384645acb663fadb4af48e9ea1228a67635",
            "namespace_name": "test",
            "pod_name": "loggen-qa-json-b9lx9"
          },
          "viaq_msg_id": "YzdkNTExZDctMzUxMS00MmY0LWJjY2MtZWM5N2MwMTFkN2Mx",
          "level": "unknown",
          "openshift": {
            "sequence": 604
          },
          "message": "{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }",
          "docker": {
            "container_id": "9974128c814a45391c8b4222ab932d24e5b2eaa98527573b950344c6c52e143e"
          },
          "hostname": "ip-10-0-148-10.us-east-2.compute.internal",
          "log_type": "application",
          "@timestamp": "2022-10-18T06:57:12.112363346+00:00",
          "pipeline_metadata": {
            "collector": {
              "received_at": "2022-10-18T06:57:12.113049+00:00",
              "name": "fluentd",
              "inputname": "fluent-plugin-systemd",
              "version": "1.14.6 1.6.0",
              "ipaddr4": "10.0.148.10"
            }
          }
        },
        "fields": {
          "@timestamp": [
            "2022-10-18T06:57:12.112Z"
          ],
          "pipeline_metadata.collector.received_at": [
            "2022-10-18T06:57:12.113Z"
          ]
        },
        "sort": [
          1666076232112
        ]
      } 

      *Switch to Vector as collector and check logs in Elasticsearch.

      {
        "_index": "app-000001",
        "_type": "_doc",
        "_id": "ZmY3YzhkZGMtMDQ5OS00ZTBiLTk3ZTctN2QzZTYyNmE3Mzgz",
        "_version": 1,
        "_score": null,
        "_source": {
          "kubernetes": {
            "container_name": "loggen-qa-json",
            "flat_labels": [
              "test=loggen-qa-json",
              "run=centos-logtest"
            ],
            "pod_ip": "10.131.0.69",
            "annotations": {
              "k8s.v1.cni.cncf.io/networks-status": "[{\n    \"name\": \"openshift-sdn\",\n    \"interface\": \"eth0\",\n    \"ips\": [\n        \"10.131.0.69\"\n    ],\n    \"default\": true,\n    \"dns\": {}\n}]",
              "seccomp.security.alpha.kubernetes.io/pod": "runtime/default",
              "openshift.io/scc": "restricted-v2",
              "k8s.v1.cni.cncf.io/network-status": "[{\n    \"name\": \"openshift-sdn\",\n    \"interface\": \"eth0\",\n    \"ips\": [\n        \"10.131.0.69\"\n    ],\n    \"default\": true,\n    \"dns\": {}\n}]"
            },
            "pod_owner": "ReplicationController/loggen-qa-json",
            "pod_id": "aa174984-2070-4a02-b0c4-f905bc4f714b",
            "namespace_labels": {
              "pod-security.kubernetes.io/warn-version": "v1.24",
              "pod-security.kubernetes.io/audit-version": "v1.24",
              "pod-security.kubernetes.io/audit": "restricted",
              "pod-security.kubernetes.io/warn": "restricted",
              "kubernetes.io/metadata.name": "test"
            },
            "container_id": "cri-o://9974128c814a45391c8b4222ab932d24e5b2eaa98527573b950344c6c52e143e",
            "container_image": "quay.io/openshifttest/ocp-logtest@sha256:16232868ba1143721b786dbabb3f7384645acb663fadb4af48e9ea1228a67635",
            "labels": {},
            "namespace_name": "test",
            "pod_name": "loggen-qa-json-b9lx9"
          },
          "level": "default",
          "message": "{\"message\": \"MERGE_JSON_LOG=true\", \"level\": \"debug\",\"Layer1\": \"layer1 0\", \"layer2\": {\"name\":\"Layer2 1\", \"tips\":\"Decide by PRESERVE_JSON_LOG\"}, \"StringNumber\":\"10\", \"Number\": 10,\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\", \"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\" }",
          "hostname": "ip-10-0-148-10.us-east-2.compute.internal",
          "log_type": "application",
          "@timestamp": "2022-10-18T07:00:07.356198158Z",
          "write_index": "app-write",
          "structured": {
            "foo:bar": "Colon Item",
            "foo.bar": "Dot Item",
            "Number": 10,
            "level": "debug",
            "{foobar}": "Brace Item",
            "foo bar": "Space Item",
            "StringNumber": "10",
            "layer2": {
              "name": "Layer2 1",
              "tips": "Decide by PRESERVE_JSON_LOG"
            },
            "message": "MERGE_JSON_LOG=true",
            "Layer1": "layer1 0",
            "[foobar]": "Bracket Item"
          }
        },
        "fields": {
          "@timestamp": [
            "2022-10-18T07:00:07.356Z"
          ]
        },
        "sort": [
          1666076407356
        ]
      } 

      Additional Notes:

      The issue is present in 5.5.3 and 5.6.0 as well.

            jcantril@redhat.com Jeffrey Cantrill
            rhn-support-ikanse Ishwar Kanse
            Ishwar Kanse Ishwar Kanse
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

              Created:
              Updated:
              Resolved: