[release-5.2] Rejected by Elasticsearch and unexpected json-parsing

Fluentd is getting a 400 - Rejected by Elasticsearch

2021-06-28T11:59:35.424914929Z 2021-06-28 11:59:35 +0000 [warn]: dump an error event: error_class=Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError error="400 - Rejected by Elasticsearch" location=nil tag="kubernetes.var.log.containers.dev-mongodb-arbiter-0_e4589f-dev_mongodb-arbiter-c067f80aa23f9a7c25515c47931d2355f1013d448c7c9161b7afa8c6c3fc5015.log" time=2021-06-28 11:59:32.389462043 +0000 record={"docker"=>

Unknown macro: {"container_id"=>"c067f80aa23f9a7c25515c47931d2355f1013d448c7c9161b7afa8c6c3fc5015"}

, "kubernetes"=>{"container_name"=>"mongodb-arbiter", "namespace_name"=>"e4589f-dev", "pod_name"=>"dev-mongodb-arbiter-0", "container_image"=>"docker-remote.artifacts.example.com/bitnami/mongodb:4.4.4-debian-10-r0", "container_image_id"=>"docker-remote.artifacts.example.com/bitnami/mongodb@sha256:95abfb776bb4e6ee34f7b5b1c811f978d132136035deacdb7143f798f0343a31", "pod_id"=>"feecd477-6575-4cce-84f4-3561a6bc5cd7", "host"=>"test.example.com", "master_url"=>"https://kubernetes.default.svc", "namespace_id"=>"96c9ee3b-b7e0-4d05-ae6f-06da77b2959c", "namespace_labels"=>

Unknown macro: {"environment"=>"dev", "name"=>"e4589f", "profile_id"=>"136", "project_type"=>"user", "provisioned-by"=>"sample", "quota"=>"small", "devops_example_ca/sample-app"=>"e4589f"}

, "flat_labels"=>["controller-revision-hash=dev-mongodb-arbiter-75bfbc6bbc", "app_kubernetes_io/component=arbiter", "app_kubernetes_io/instance=dev", "app_kubernetes_io/managed-by=Helm", "app_kubernetes_io/name=mongodb", "helm_sh/chart=mongodb-10.7.1", "statefulset_kubernetes_io/pod-name=dev-mongodb-arbiter-0"]}, "message"=>{"t"=>

Unknown macro: {"$date"=>"2021-06-28T11}

, "s"=>"I", "c"=>"NETWORK", "id"=>22944, "ctx"=>"conn245040", "msg"=>"Connection ended", "attr"=>{"remote"=>"10.97.50.1:54034", "connectionId"=>245040, "connectionCount"=>12}}, "level"=>"unknown", "hostname"=>"test.example.com", "pipeline_metadata"=>{"collector"=>{"ipaddr4"=>"142.34.151.161", "inputname"=>"fluent-plugin-systemd", "name"=>"fluentd", "received_at"=>"2021-06-28T11:59:32.429777+00:00", "version"=>"1.7.4 1.6.0"}}, "@timestamp"=>"2021-06-28T11:59:32.389462+00:00", "viaq_index_name"=>"app-write", "viaq_msg_id"=>"MDRhNTllOGItOTcwMS00MjZiLTllY2EtYzc4ZDUwMjEwZGRk"}

Taking record, replacing => with :, piping to jq:

{
"docker":

Unknown macro: { "container_id"}

,
"kubernetes": {
"container_name": "mongodb-arbiter",
"namespace_name": "e4589f-dev",
"pod_name": "dev-mongodb-arbiter-0",
"container_image": "docker-remote.artifacts.example.com/bitnami/mongodb:4.4.4-debian-10-r0",
"container_image_id": "docker-remote.artifacts.example.com/bitnami/mongodb@sha256:95abfb776bb4e6ee34f7b5b1c811f978d132136035deacdb7143f798f0343a31",
"pod_id": "feecd477-6575-4cce-84f4-3561a6bc5cd7",
"host": "mcs-silver-app-40.dmz",
"master_url": "https://kubernetes.default.svc",
"namespace_id": "96c9ee3b-b7e0-4d05-ae6f-06da77b2959c",
"namespace_labels":

Unknown macro: { "environment"}

,
"flat_labels": [
"controller-revision-hash=dev-mongodb-arbiter-75bfbc6bbc",
"app_kubernetes_io/component=arbiter",
"app_kubernetes_io/instance=dev",
"app_kubernetes_io/managed-by=Helm",
"app_kubernetes_io/name=mongodb",
"helm_sh/chart=mongodb-10.7.1",
"statefulset_kubernetes_io/pod-name=dev-mongodb-arbiter-0"
]
},
"message": {
"t":

Unknown macro: { "$date"}

,
"s": "I",
"c": "NETWORK",
"id": 22944,
"ctx": "conn245040",
"msg": "Connection ended",
"attr":

Unknown macro: { "remote"}

},
"level": "unknown",
"hostname": "mcs-silver-app-40.dmz",
"pipeline_metadata": {
"collector":

Unknown macro: { "ipaddr4"}

},
"@timestamp": "2021-06-28T11:59:32.389462+00:00",
"viaq_index_name": "app-write",
"viaq_msg_id": "MDRhNTllOGItOTcwMS00MjZiLTllY2EtYzc4ZDUwMjEwZGRk"
}

Issue is occurring on 4.6 where MERGE_JSON_LOG shouldn't be available and is not enabled. Will upload must-gather in private attachment.