Show
1) Create a CLF instance without adding "outputDefaults" fields
spec:
pipelines:
- inputRefs:
- application
outputRefs:
- default
parse: json
2) Deploy an application and send JSON logs
$ oc new-app rails-postgresql-example
$ oc rsh rails-postgresql-example-1-7wcnz
sh-4.4$ echo '{ "level" : "info" , "name" : "fred" , "home" : "bedrock" }' >> /proc/1/fd/1
sh-4.4$ echo '{ "level" : "info" , "name" : "fred" , "home" : "bedrock" }' >> /proc/1/fd/1
sh-4.4$ echo '{ "level" : "info" , "name" : "fred" , "home" : "bedrock" }' >> /proc/1/fd/1
sh-4.4$ echo '{ "level" : "info" , "name" : "fred" , "home" : "bedrock" }' >> /proc/1/fd/1
3) Check in Kibana that the log is not parsed:
{
"_index" : "app-000001" ,
"_type" : "_doc" ,
"_id" : "MDc4ZmQ4OTYtNmUxYi00ZGMxLTkwNDctNWUwZmE5YzA5OTVl" ,
"_version" : 1,
"_score" : null ,
"_source" : {
"kubernetes" : {
"container_image_id" : "image-registry.openshift-image-registry.svc:5000/adritest/rails-postgresql-example@sha256:e87ed7ea43dbf33b3b54fec51f189511df92cf701a928360319a5089009ceb09" ,
"container_name" : "rails-postgresql-example" ,
"namespace_id" : "13ad3465-9a76-43a4-9c1c-468896a44c88" ,
"flat_labels" : [
"deployment=rails-postgresql-example-1" ,
"deploymentconfig=rails-postgresql-example" ,
"name=rails-postgresql-example"
],
"pod_ip" : "10.128.2.15" ,
"host" : "worker-2.adritest.lab.psi.pnq2.redhat.com" ,
"master_url" : "https: //kubernetes. default .svc" ,
"pod_id" : "aea407ab-5351-4d65-9653-4e21a6cebee2" ,
"namespace_labels" : {
"pod-security.kubernetes.io/warn-version" : "v1.24" ,
"pod-security.kubernetes.io/audit-version" : "v1.24" ,
"pod-security.kubernetes.io/audit" : "restricted" ,
"pod-security.kubernetes.io/warn" : "restricted" ,
"kubernetes.io/metadata.name" : "adritest"
},
"container_image" : "image-registry.openshift-image-registry.svc:5000/adritest/rails-postgresql-example@sha256:e87ed7ea43dbf33b3b54fec51f189511df92cf701a928360319a5089009ceb09" ,
"namespace_name" : "adritest" ,
"pod_name" : "rails-postgresql-example-1-7wcnz"
},
"viaq_msg_id" : "MDc4ZmQ4OTYtNmUxYi00ZGMxLTkwNDctNWUwZmE5YzA5OTVl" ,
"level" : "info" ,
"openshift" : {
"sequence" : 97
},
"message" : "{\" level\ ":\" info\ ",\" name\ ":\" fred\ ",\" home\ ":\" bedrock\ "}" ,
"docker" : {
"container_id" : "1b54c92a8302f6789ac8fcc23d4d8f0b87f17890d5cb63a24172ac02bf3fb8e8"
},
"hostname" : "worker-2.adritest.lab.psi.pnq2.redhat.com" ,
"log_type" : "application" ,
"@timestamp" : "2022-12-01T12:21:44.368172331+00:00" ,
"pipeline_metadata" : {
"collector" : {
"received_at" : "2022-12-01T12:21:44.368844+00:00" ,
"name" : "fluentd" ,
"inputname" : "fluent-plugin-systemd" ,
"version" : "1.14.6 1.6.0" ,
"ipaddr4" : "10.74.210.205"
}
}
},
"fields" : {
"@timestamp" : [
"2022-12-01T12:21:44.368Z"
],
"pipeline_metadata.collector.received_at" : [
"2022-12-01T12:21:44.368Z"
]
},
"sort" : [
1669897304368
]
}
Additionally, modifying the CLF instance:
spec:
inputs:
- application:
namespaces:
- adritest
name: my-parsed-app-logs
outputDefaults:
elasticsearch:
structuredTypeKey: openshift.labels.myLabel
pipelines:
- inputRefs:
- application
labels:
myLabel: ParsedApp
outputRefs:
- default
parse: json
And repeating steps 2 and 3, we can observe that the logs are parsed properly in Kibana (check the structured message ).
{
"_index" : "app-parsedapp-000001" ,
"_type" : "_doc" ,
"_id" : "NTJlODZkMDgtNDA2NC00ODM3LTlmZDMtMGE1NDIwOTc1MDU5" ,
"_version" : 1,
"_score" : null ,
"_source" : {
"kubernetes" : {
"container_image_id" : "image-registry.openshift-image-registry.svc:5000/adritest/rails-postgresql-example@sha256:e87ed7ea43dbf33b3b54fec51f189511df92cf701a928360319a5089009ceb09" ,
"container_name" : "rails-postgresql-example" ,
"namespace_id" : "13ad3465-9a76-43a4-9c1c-468896a44c88" ,
"flat_labels" : [
"deployment=rails-postgresql-example-1" ,
"deploymentconfig=rails-postgresql-example" ,
"name=rails-postgresql-example"
],
"pod_ip" : "10.128.2.15" ,
"host" : "worker-2.adritest.lab.psi.pnq2.redhat.com" ,
"master_url" : "https: //kubernetes. default .svc" ,
"pod_id" : "aea407ab-5351-4d65-9653-4e21a6cebee2" ,
"namespace_labels" : {
"pod-security.kubernetes.io/warn-version" : "v1.24" ,
"pod-security.kubernetes.io/audit-version" : "v1.24" ,
"pod-security.kubernetes.io/audit" : "restricted" ,
"pod-security.kubernetes.io/warn" : "restricted" ,
"kubernetes.io/metadata.name" : "adritest"
},
"container_image" : "image-registry.openshift-image-registry.svc:5000/adritest/rails-postgresql-example@sha256:e87ed7ea43dbf33b3b54fec51f189511df92cf701a928360319a5089009ceb09" ,
"namespace_name" : "adritest" ,
"pod_name" : "rails-postgresql-example-1-7wcnz"
},
"viaq_msg_id" : "NTJlODZkMDgtNDA2NC00ODM3LTlmZDMtMGE1NDIwOTc1MDU5" ,
"level" : "info" ,
"openshift" : {
"labels" : {
"myLabel" : "ParsedApp"
}
},
"docker" : {
"container_id" : "1b54c92a8302f6789ac8fcc23d4d8f0b87f17890d5cb63a24172ac02bf3fb8e8"
},
"hostname" : "worker-2.adritest.lab.psi.pnq2.redhat.com" ,
"log_type" : "application" ,
"@timestamp" : "2022-12-01T12:14:41.722003634+00:00" ,
"pipeline_metadata" : {
"collector" : {
"received_at" : "2022-12-01T12:14:41.722891+00:00" ,
"name" : "fluentd" ,
"inputname" : "fluent-plugin-systemd" ,
"version" : "1.14.6 1.6.0" ,
"ipaddr4" : "10.74.210.205"
}
},
"structured" : {
"level" : "info" ,
"name" : "fred" ,
"home" : "bedrock"
}
},
"fields" : {
"@timestamp" : [
"2022-12-01T12:14:41.722Z"
],
"pipeline_metadata.collector.received_at" : [
"2022-12-01T12:14:41.722Z"
]
},
"sort" : [
1669896881722
]
}