-
Bug
-
Resolution: Obsolete
-
Undefined
-
Logging 5.1
Version:
clusterserviceversion.operators.coreos.com/cluster-logging.5.1.0-26
clusterserviceversion.operators.coreos.com/elasticsearch-operator.5.1.0-34
Plus
elasticsearch with openshift-ingest-plugin
$ oc exec c elasticsearch elasticsearch-cdm-wd1u5qo4-1-5c6b7b8f65-5gfrj – es_util '-query=_cat/plugins'
elasticsearch-cdm-wd1u5qo4-1 opendistro_security 0.10.0.4
elasticsearch-cdm-wd1u5qo4-1 openshift-ingest-plugin 6.8.1.0
elasticsearch-cdm-wd1u5qo4-1 prometheus-exporter 6.8.1.0
How reproducible: Always
Steps to Reproduce:
1. Deploy clusterlogging
2. Create 50 projects and create apps under these namespaces.
oc new-project app-1, ..... oc new-project app-50 oc new-app https://raw.githubusercontent.com/openshift/verification-tests/master/testdata/logging/loggen/container_json_log_template.json
3. Forward to defaut ES using structuredIndexKey: kubernetes.namespace_name
cat <<EOF|oc create -f - apiVersion: logging.openshift.io/v1 kind: ClusterLogForwarder metadata: name: instance spec: inputs: - application: namespaces: - app-1 name: input-qa1 - application: namespaces: - app-2 name: input-qa2 ......... - application: namespaces: - app-50 name: input-qa50 outputs : - elasticsearch: structuredIndexKey: kubernetes.namespace_name name: default-es-a secret: name: fluentd type: elasticsearch url: https://elasticsearch.openshift-logging.svc:9200 pipelines: - inputRefs: - input-qa1 - input-qa2 ............ - input-qa50 name: pipeline-a outputRefs: - default-es-a parse: json EOF
4. Check the es pod logs
[2021-05-20T06:32:48,828][WARN ][r.suppressed ] [elasticsearch-cdm-amcmfcdw-1] path: /_template/common.*,ocp-gen-*, params: {name=common.*,ocp-gen-*}[2021-05-20T06:32:48,828][WARN ][r.suppressed ] [elasticsearch-cdm-amcmfcdw-1] path: /_template/common.*,ocp-gen-*, params: {name=common.*,ocp-gen-*}org.elasticsearch.transport.RemoteTransportException: [elasticsearch-cdm-amcmfcdw-2][10.131.0.31:9300][indices:admin/template/get]Caused by: org.elasticsearch.ElasticsearchSecurityException: Unexpected exception indices:admin/template/get at com.amazon.opendistroforelasticsearch.security.filter.OpenDistroSecurityFilter.apply0(OpenDistroSecurityFilter.java:274) ~[?:?] at com.amazon.opendistroforelasticsearch.security.filter.OpenDistroSecurityFilter.apply(OpenDistroSecurityFilter.java:119) ~[?:?] at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:165) ~[elasticsearch-6.8.1.jar:6.8.1.redhat-00006] at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:139) ~[elasticsearch-6.8.1.jar:6.8.1.redhat-00006] at org.elasticsearch.action.support.HandledTransportAction$TransportHandler.messageReceived(HandledTransportAction.java:89) ~[elasticsearch-6.8.1.jar:6.8.1.redhat-00006] at org.elasticsearch.action.support.HandledTransportAction$TransportHandler.messageReceived(HandledTransportAction.java:80) ~[elasticsearch-6.8.1.jar:6.8.1.redhat-00006] at com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceivedDecorate(OpenDistroSecuritySSLRequestHandler.java:194) ~[?:?] at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityRequestHandler.messageReceivedDecorate(OpenDistroSecurityRequestHandler.java:284) ~[?:?] at com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceived(OpenDistroSecuritySSLRequestHandler.java:166) ~[?:?] at com.amazon.opendistroforelasticsearch.security.OpenDistroSecurityPlugin$7$1.messageReceived(OpenDistroSecurityPlugin.java:646) ~[?:?] at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) ~[elasticsearch-6.8.1.jar:6.8.1.redhat-00006] at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1087) ~[elasticsearch-6.8.1.jar:6.8.1.redhat-00006] at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-6.8.1.jar:6.8.1.redhat-00006] at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:193) ~[elasticsearch-6.8.1.jar:6.8.1.redhat-00006] at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1046) ~[elasticsearch-6.8.1.jar:6.8.1.redhat-00006] at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:932) ~[elasticsearch-6.8.1.jar:6.8.1.redhat-00006] at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:763) ~[elasticsearch-6.8.1.jar:6.8.1.redhat-00006] at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1436) ~[?:?] at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1203) ~[?:?] at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1247) ~[?:?] at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:502) ~[?:?] at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:441) ~[?:?] at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:278) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] at java.lang.Thread.run(Thread.java:829) ~[?:?]
Attached the complete ES log.