[release-5.6] ElasticsearchError error="400 - Rejected by Elasticsearch" when adding some labels in application namespaces

Before this change, records written to Elasticsearch would fail to be ingested if multiple label keys had the same prefix and some keys included included dots. This fixes the limitation of Elasticsearch by replacing dots in the label keys with underscores.

1) Create a new project $ oc create project adri 2) Label the namespace $ oc label namespace adri "app=config-server" $ oc label namespace adri "app.kubernetes.io/instance=config-server-uat" $ oc label namespace adri "app.test=test" 3) Deploy an application $ oc new -app rails-postgresql-example 4) Check collector logs 2022-11-18 11:25:08 +0000 [warn]: dump an error event: error_class=Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError error= "400 - Rejected by Elasticsearch" location=nil tag= "kubernetes. var .log.pods.adri_rails-postgresql-example-1-build_bc8495af-acd6-4da3-bede-3d4b47ff99fb.sti-build.0.log" time=2022-11-18 11:25:08.027204286 +0000 record={ "@timestamp" => "2022-11-18T11:25:08.027204286+00:00" , "message" => "Push successful" , "docker" =>{ "container_id" => "f822a2fada8223e39e4a6d26dca41ec21e52c10ce39a3146ec9a6e7b68b860ae" }, "kubernetes" =>{ "container_name" => "sti-build" , "namespace_name" => "adri" , "pod_name" => "rails-postgresql-example-1-build" , "container_image" => "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c043c4d68753120a32044f476a8e1977101139cbab3694761ef523da5be8fb0e" , "container_image_id" => "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c043c4d68753120a32044f476a8e1977101139cbab3694761ef523da5be8fb0e" , "pod_id" => "bc8495af-acd6-4da3-bede-3d4b47ff99fb" , "pod_ip" => "10.129.2.80" , "host" => "worker-2.adricluster.lab.psi.pnq2.redhat.com" , "master_url" => "https: //kubernetes. default .svc" , "namespace_id" => "b2c972e1-05ea-412d-a2e9-8db4bb800c94" , "namespace_labels" =>{ "app" => "config-server" , "app.kubernetes.io/instance" => "config-server-uat" , "app.test" => "test" , "kubernetes.io/metadata.name" => "adri" , "pod-security.kubernetes.io/audit" => "restricted" , "pod-security.kubernetes.io/audit-version" => "v1.24" , "pod-security.kubernetes.io/warn" => "restricted" , "pod-security.kubernetes.io/warn-version" => "v1.24" }, "flat_labels" =>[ "openshift.io/build.name=rails-postgresql-example-1" ]}, "level" => "unknown" , "hostname" => "worker-2.adricluster.lab.psi.pnq2.redhat.com" , "pipeline_metadata" =>{ "collector" =>{ "ipaddr4" => "10.74.215.95" , "inputname" => "fluent-plugin-systemd" , "name" => "fluentd" , "received_at" => "2022-11-18T11:25:08.028075+00:00" , "version" => "1.14.6 1.6.0" }}, "openshift" =>{ "sequence" =>2598}, "viaq_msg_id" => "NGY0YzNlMmMtNWNjZS00ODZlLWJkNWItYWQ3MzE0OWNkYmRl" , "log_type" => "application" , "viaq_index_name" => "app-write" }