whoami: cannot find name for user ID 1001020000 Running as uid=1001020000 gid=0(root) groups=0(root),1001020000: PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/s2i LD_LIBRARY_PATH= PKG_CONFIG_PATH= XDG_DATA_DIRS= Running in Kubernetes Registering internal (pod) endpoint only PORT_ARGS=-p 8080 -r 8080 WATSON_SERVICE_ADDRESS=10.128.20.148 Registering instance id (from Kubernetes pod name) as "7dd597-z8vbd" WARNING: MM_SVC_GRPC_PRIVATE_KEY_PATH not set *AND/OR* MM_ENABLE_SSL=false, using PLAINTEXT for internal comms SERVICE_VERSION= build-version=model-serving-qe Registering ModelMesh Service version as "model-serving-qe" JAVA_HOME set to /usr/lib/jvm/jre-17-openjdk Java version information: openjdk version "17.0.4" 2022-07-19 LTS OpenJDK Runtime Environment (Red_Hat-17.0.4.0.8-2.el8_6) (build 17.0.4+8-LTS) OpenJDK 64-Bit Server VM (Red_Hat-17.0.4.0.8-2.el8_6) (build 17.0.4+8-LTS, mixed mode, sharing) KV_STORE=etcd:/opt/kserve/mmesh/etcd/etcd_connection LL_REGISTRY= ZOOKEEPER= WATSON_SERVICE_ADDRESS=10.128.20.148 MM_SERVICE_NAME=modelmesh-serving PRIVATE_ENDPOINT= MM_LOCATION=10.0.244.168 MM_SERVICE_CLASS=com.ibm.watson.modelmesh.SidecarModelMesh INTERNAL_GRPC_PORT=8085 SERVICE_ARGS=-p 8080 -r 8080 -i 7dd597-z8vbd -v model-serving-qe Certificate was added to keystore Imported Kubernetes CA certificate into litelinks truststore: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt Using process mem limit of 448MiB from /sys/fs/cgroup/memory/memory.limit_in_bytes MEM_LIMIT_MB=448 Using default heap size of MIN(41% of MEM_LIMIT_MB, 640MiB) = 183MiB HEAP_SIZE_MB=183 MAX_GC_PAUSE=50 millisecs MEM_HEADROOM_MB=78 MAX_DIRECT_BUFS_MB=187 Health probe HTTP endpoint will use port 8089 SHUTDOWN_TIMEOUT_MS=90000 + exec /usr/lib/jvm/jre-17-openjdk/bin/java -cp 'lib/litelinks-core-1.7.2.jar:lib/*' -XX:+UnlockExperimentalVMOptions -XX:+UseG1GC -XX:MaxGCPauseMillis=50 -XX:-ResizePLAB -Xmx183m -Xms183m -XX:MaxInlineLevel=28 -Xlog:gc:/opt/kserve/mmesh/log/vgc_modelmesh-serving-model-server-luca-test-dfd7dd597-z8vbd.log -Dfile.encoding=UTF8 -Dio.netty.tryReflectionSetAccessible=true --add-opens=java.base/java.nio=ALL-UNNAMED -Dcom.redhat.fips=false -XX:MaxDirectMemorySize=20971520 -Dio.netty.maxDirectMemory=196083712 -Dio.grpc.netty.useCustomAllocator=false -Dwatson.ssl.truststore.path=/opt/kserve/mmesh/lib/truststore.jks -Dwatson.ssl.truststore.password=watson15qa -Dlitelinks.cancel_on_client_close=true -Dlitelinks.threadcontexts=log_mdc -Dlitelinks.shutdown_timeout_ms=90000 -Dlitelinks.produce_pooled_bytebufs=true -Dlitelinks.ssl.use_jdk=false -Dlog4j.configurationFile=/opt/kserve/mmesh/lib/log4j2.xml -Dlog4j2.enable.threadlocals=true -Dlog4j2.garbagefree.threadContextMap=true com.ibm.watson.litelinks.server.LitelinksService -s com.ibm.watson.modelmesh.SidecarModelMesh -n modelmesh-serving -a /opt/kserve/mmesh/model-mesh.anchor -p 8080 -r 8080 -i 7dd597-z8vbd -v model-serving-qe -h 8089 using service registry string: etcd:/opt/kserve/mmesh/etcd/etcd_connection {"instant":{"epochSecond":1669649040,"nanoOfSecond":348187543},"thread":"main","level":"INFO","loggerName":"com.ibm.watson.litelinks.server.WatchedService","message":"Starting service-watching wrapper; hostname=10.128.20.148","endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":1,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":404269298},"thread":"main","level":"INFO","loggerName":"com.ibm.watson.litelinks.server.ProbeHttpServer","message":"Starting litelinks health probe http server on port 8089","endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":1,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":408966680},"thread":"main","level":"INFO","loggerName":"com.ibm.watson.litelinks.NettyCommon","message":"Litelinks using native transport (epoll)","endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":1,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":428344438},"thread":"main","level":"INFO","loggerName":"com.ibm.watson.litelinks.NettyCommon","message":"Creating litelinks shared worker ELG with 3 threads","endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":1,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":456722541},"thread":"ll-svc-events-3","level":"INFO","loggerName":"com.ibm.watson.litelinks.server.DefaultThriftServer","message":"initializing service implementation...","endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":20,"threadPriority":5} service starting, type "stop" to stop {"instant":{"epochSecond":1669649040,"nanoOfSecond":623426715},"thread":"ll-svc-events-3","level":"INFO","loggerName":"com.ibm.watson.kvutils.factory.KVUtilsFactory","message":"KV_STORE=etcd:/opt/kserve/mmesh/etcd/etcd_connection","endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":20,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":623908333},"thread":"ll-svc-events-3","level":"INFO","loggerName":"com.ibm.watson.kvutils.factory.KVUtilsFactory","message":"creating new etcd KV factory with config file: /opt/kserve/mmesh/etcd/etcd_connection","endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":20,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":738234062},"thread":"ll-svc-events-3","level":"INFO","loggerName":"com.ibm.watson.modelmesh.ModelMesh","message":"Service name is modelmesh-serving, Instance ID is 7dd597-z8vbd, Version is model-serving-qe","endOfBatch":false,"loggerFqcn":"org.apache.logging.log4j.spi.AbstractLogger","contextMap":{},"threadId":20,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":739066301},"thread":"ll-svc-events-3","level":"WARN","loggerName":"com.ibm.watson.modelmesh.ModelMesh","message":"Unrecognized service version format: model-serving-qe","endOfBatch":false,"loggerFqcn":"org.apache.logging.log4j.spi.AbstractLogger","contextMap":{},"threadId":20,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":766290046},"thread":"ll-svc-events-2","level":"INFO","loggerName":"com.ibm.watson.litelinks.etcd.EtcdWatchedService","message":"created etcd client and initiated session lease","endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":19,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":767320572},"thread":"ll-svc-events-3","level":"INFO","loggerName":"com.ibm.watson.modelmesh.ModelMesh","message":"Heap mem info: init = 192937984(188416K) used = 15323272(14964K) committed = 192937984(188416K) max = 192937984(188416K)","endOfBatch":false,"loggerFqcn":"org.apache.logging.log4j.spi.AbstractLogger","contextMap":{},"threadId":20,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":767835785},"thread":"ll-svc-events-3","level":"INFO","loggerName":"com.ibm.watson.modelmesh.ModelMesh","message":"Non-heap mem info: init = 7667712(7488K) used = 39355336(38432K) committed = 42729472(41728K) max = -1(-1K)","endOfBatch":false,"loggerFqcn":"org.apache.logging.log4j.spi.AbstractLogger","contextMap":{},"threadId":20,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":770239578},"thread":"ll-svc-events-3","level":"INFO","loggerName":"com.ibm.watson.modelmesh.ModelMesh","message":"Max heap = 0.180GiB, baseline overhead = 14MiB","endOfBatch":false,"loggerFqcn":"org.apache.logging.log4j.spi.AbstractLogger","contextMap":{},"threadId":20,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":770682147},"thread":"ll-svc-events-3","level":"INFO","loggerName":"com.ibm.watson.modelmesh.ModelMesh","message":"Instance zone = , location = 10.0.244.168","endOfBatch":false,"loggerFqcn":"org.apache.logging.log4j.spi.AbstractLogger","contextMap":{},"threadId":20,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":772347654},"thread":"ll-svc-events-3","level":"INFO","loggerName":"com.ibm.watson.modelmesh.ModelMesh","message":"6 instance labels configured: [mt:onnx, mt:onnx:1, mt:openvino_ir, mt:openvino_ir:opset1, pv:grpc-v1, rt:model-server-luca-test]","endOfBatch":false,"loggerFqcn":"org.apache.logging.log4j.spi.AbstractLogger","contextMap":{},"threadId":20,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":780514509},"thread":"ll-svc-events-3","level":"INFO","loggerName":"com.ibm.watson.modelmesh.VModelManager","message":"Owner for all created vmodels will be \"ksp\"","endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":20,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":820882891},"thread":"etcd-event-pool-0","level":"INFO","loggerName":"com.ibm.etcd.client.GrpcClient","message":"Retryable onError #1 on underlying stream of method etcdserverpb.Lease/LeaseKeepAlive: io.grpc.StatusRuntimeException: UNAVAILABLE: io exception","endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":24,"threadPriority":5} {"instant":{"epochSecond":1669649040,"nanoOfSecond":822390510},"thread":"etcd-event-pool-0","level":"INFO","loggerName":"com.ibm.etcd.client.GrpcClient","message":"Retryable onError #2 on underlying stream of method etcdserverpb.Lease/LeaseKeepAlive: io.grpc.StatusRuntimeException: UNAVAILABLE: io exception","endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":24,"threadPriority":5} {"instant":{"epochSecond":1669649041,"nanoOfSecond":553420016},"thread":"etcd-event-pool-0","level":"INFO","loggerName":"com.ibm.etcd.client.GrpcClient","message":"Retryable onError #3 on underlying stream of method etcdserverpb.Lease/LeaseKeepAlive: io.grpc.StatusRuntimeException: UNAVAILABLE: io exception","endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":24,"threadPriority":5} {"instant":{"epochSecond":1669649041,"nanoOfSecond":709176292},"thread":"ll-svc-events-5","level":"ERROR","loggerName":"com.ibm.watson.litelinks.server.DefaultThriftServer","message":"service implementation initialization failed","thrown":{"commonElementCount":0,"localizedMessage":"Problem connecting to etcd","message":"Problem connecting to etcd","name":"java.lang.Exception","cause":{"commonElementCount":0,"localizedMessage":"etcd connection verification failed","message":"etcd connection verification failed","name":"java.lang.RuntimeException","cause":{"commonElementCount":0,"localizedMessage":"UNAVAILABLE: io exception","message":"UNAVAILABLE: io exception","name":"io.grpc.StatusRuntimeException","cause":{"commonElementCount":0,"localizedMessage":"finishConnect(..) failed: Connection refused: etcd.redhat-ods-applications/172.30.30.132:2379","message":"finishConnect(..) failed: Connection refused: etcd.redhat-ods-applications/172.30.30.132:2379","name":"io.netty.channel.AbstractChannel$AnnotatedConnectException","cause":{"commonElementCount":0,"localizedMessage":"finishConnect(..) failed: Connection refused","message":"finishConnect(..) failed: Connection refused","name":"java.net.ConnectException","extendedStackTrace":[{"class":"io.netty.channel.unix.Errors","method":"newConnectException0","file":"Errors.java","line":155,"exact":false,"location":"netty-transport-native-unix-common-4.1.84.Final.jar","version":"4.1.84.Final"},{"class":"io.netty.channel.unix.Errors","method":"handleConnectErrno","file":"Errors.java","line":128,"exact":false,"location":"netty-transport-native-unix-common-4.1.84.Final.jar","version":"4.1.84.Final"},{"class":"io.netty.channel.unix.Socket","method":"finishConnect","file":"Socket.java","line":359,"exact":false,"location":"netty-transport-native-unix-common-4.1.84.Final.jar","version":"4.1.84.Final"},{"class":"io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe","method":"doFinishConnect","file":"AbstractEpollChannel.java","line":710,"exact":false,"location":"netty-transport-classes-epoll-4.1.84.Final.jar","version":"4.1.84.Final"},{"class":"io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe","method":"finishConnect","file":"AbstractEpollChannel.java","line":687,"exact":false,"location":"netty-transport-classes-epoll-4.1.84.Final.jar","version":"4.1.84.Final"},{"class":"io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe","method":"epollOutReady","file":"AbstractEpollChannel.java","line":567,"exact":false,"location":"netty-transport-classes-epoll-4.1.84.Final.jar","version":"4.1.84.Final"},{"class":"io.netty.channel.epoll.EpollEventLoop","method":"processReady","file":"EpollEventLoop.java","line":489,"exact":false,"location":"netty-transport-classes-epoll-4.1.84.Final.jar","version":"4.1.84.Final"},{"class":"io.netty.channel.epoll.EpollEventLoop","method":"run","file":"EpollEventLoop.java","line":397,"exact":false,"location":"netty-transport-classes-epoll-4.1.84.Final.jar","version":"4.1.84.Final"},{"class":"io.netty.util.concurrent.SingleThreadEventExecutor$4","method":"run","file":"SingleThreadEventExecutor.java","line":997,"exact":false,"location":"netty-common-4.1.84.Final.jar","version":"4.1.84.Final"},{"class":"io.netty.util.internal.ThreadExecutorMap$2","method":"run","file":"ThreadExecutorMap.java","line":74,"exact":false,"location":"netty-common-4.1.84.Final.jar","version":"4.1.84.Final"},{"class":"io.netty.util.concurrent.FastThreadLocalRunnable","method":"run","file":"FastThreadLocalRunnable.java","line":30,"exact":false,"location":"netty-common-4.1.84.Final.jar","version":"4.1.84.Final"},{"class":"java.lang.Thread","method":"run","file":"Thread.java","line":833,"exact":false,"location":"?","version":"?"}]}},"extendedStackTrace":[{"class":"io.grpc.Status","method":"asRuntimeException","file":"Status.java","line":539,"exact":false,"location":"grpc-api-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.stub.ClientCalls$UnaryStreamToFuture","method":"onClose","file":"ClientCalls.java","line":544,"exact":false,"location":"grpc-stub-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.internal.ClientCallImpl","method":"closeObserver","file":"ClientCallImpl.java","line":563,"exact":false,"location":"grpc-core-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.internal.ClientCallImpl","method":"access$300","file":"ClientCallImpl.java","line":70,"exact":false,"location":"grpc-core-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed","method":"runInternal","file":"ClientCallImpl.java","line":744,"exact":false,"location":"grpc-core-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed","method":"runInContext","file":"ClientCallImpl.java","line":723,"exact":false,"location":"grpc-core-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.internal.ContextRunnable","method":"run","file":"ContextRunnable.java","line":37,"exact":false,"location":"grpc-core-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.internal.SerializingExecutor","method":"run","file":"SerializingExecutor.java","line":133,"exact":false,"location":"grpc-core-1.50.2.jar","version":"1.50.2"},{"class":"java.util.concurrent.ThreadPoolExecutor","method":"runWorker","file":"ThreadPoolExecutor.java","line":1136,"exact":false,"location":"?","version":"?"},{"class":"java.util.concurrent.ThreadPoolExecutor$Worker","method":"run","file":"ThreadPoolExecutor.java","line":635,"exact":false,"location":"?","version":"?"},{"class":"java.lang.Thread","method":"run","file":"Thread.java","line":833,"exact":false,"location":"?","version":"?"}]},"extendedStackTrace":[{"class":"com.ibm.watson.etcd.EtcdUtilsFactory","method":"lambda$verifyKvStoreConnection$1","file":"EtcdUtilsFactory.java","line":172,"exact":false,"location":"kv-utils-0.5.1.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractCatchingFuture$CatchingFuture","method":"doFallback","file":"AbstractCatchingFuture.java","line":234,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractCatchingFuture$CatchingFuture","method":"doFallback","file":"AbstractCatchingFuture.java","line":222,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractCatchingFuture","method":"run","file":"AbstractCatchingFuture.java","line":133,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.DirectExecutor","method":"execute","file":"DirectExecutor.java","line":31,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractFuture","method":"executeListener","file":"AbstractFuture.java","line":1270,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractFuture","method":"complete","file":"AbstractFuture.java","line":1038,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractFuture","method":"setException","file":"AbstractFuture.java","line":808,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractTransformFuture","method":"run","file":"AbstractTransformFuture.java","line":104,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.DirectExecutor","method":"execute","file":"DirectExecutor.java","line":31,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractFuture","method":"executeListener","file":"AbstractFuture.java","line":1270,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractFuture","method":"complete","file":"AbstractFuture.java","line":1038,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractFuture","method":"setFuture","file":"AbstractFuture.java","line":851,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractCatchingFuture$AsyncCatchingFuture","method":"setResult","file":"AbstractCatchingFuture.java","line":214,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractCatchingFuture$AsyncCatchingFuture","method":"setResult","file":"AbstractCatchingFuture.java","line":190,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractCatchingFuture","method":"run","file":"AbstractCatchingFuture.java","line":142,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.DirectExecutor","method":"execute","file":"DirectExecutor.java","line":31,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractFuture","method":"executeListener","file":"AbstractFuture.java","line":1270,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractFuture","method":"complete","file":"AbstractFuture.java","line":1038,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractFuture","method":"setException","file":"AbstractFuture.java","line":808,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"io.grpc.stub.ClientCalls$GrpcFuture","method":"setException","file":"ClientCalls.java","line":574,"exact":false,"location":"grpc-stub-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.stub.ClientCalls$UnaryStreamToFuture","method":"onClose","file":"ClientCalls.java","line":544,"exact":false,"location":"grpc-stub-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.internal.ClientCallImpl","method":"closeObserver","file":"ClientCallImpl.java","line":563,"exact":false,"location":"grpc-core-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.internal.ClientCallImpl","method":"access$300","file":"ClientCallImpl.java","line":70,"exact":false,"location":"grpc-core-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed","method":"runInternal","file":"ClientCallImpl.java","line":744,"exact":false,"location":"grpc-core-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed","method":"runInContext","file":"ClientCallImpl.java","line":723,"exact":false,"location":"grpc-core-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.internal.ContextRunnable","method":"run","file":"ContextRunnable.java","line":37,"exact":false,"location":"grpc-core-1.50.2.jar","version":"1.50.2"},{"class":"io.grpc.internal.SerializingExecutor","method":"run","file":"SerializingExecutor.java","line":133,"exact":false,"location":"grpc-core-1.50.2.jar","version":"1.50.2"},{"class":"java.util.concurrent.ThreadPoolExecutor","method":"runWorker","file":"ThreadPoolExecutor.java","line":1136,"exact":false,"location":"?","version":"?"},{"class":"java.util.concurrent.ThreadPoolExecutor$Worker","method":"run","file":"ThreadPoolExecutor.java","line":635,"exact":false,"location":"?","version":"?"},{"class":"java.lang.Thread","method":"run","file":"Thread.java","line":833,"exact":false,"location":"?","version":"?"}]},"extendedStackTrace":[{"class":"com.ibm.watson.modelmesh.ModelMesh","method":"initialize","file":"ModelMesh.java","line":617,"exact":false,"location":"model-mesh-0.4.2-SNAPSHOT.jar","version":"?"},{"class":"com.ibm.watson.litelinks.server.ThriftService","method":"doStart","file":"ThriftService.java","line":123,"exact":false,"location":"litelinks-core-1.7.2.jar","version":"?"},{"class":"com.google.common.util.concurrent.AbstractService","method":"startAsync","file":"AbstractService.java","line":251,"exact":false,"location":"guava-31.1-jre.jar","version":"?"},{"class":"com.ibm.watson.litelinks.server.DefaultThriftServer","method":"lambda$doStart$0","file":"DefaultThriftServer.java","line":390,"exact":false,"location":"litelinks-core-1.7.2.jar","version":"?"},{"class":"java.util.concurrent.ThreadPoolExecutor","method":"runWorker","file":"ThreadPoolExecutor.java","line":1136,"exact":false,"location":"?","version":"?"},{"class":"java.util.concurrent.ThreadPoolExecutor$Worker","method":"run","file":"ThreadPoolExecutor.java","line":635,"exact":false,"location":"?","version":"?"},{"class":"java.lang.Thread","method":"run","file":"Thread.java","line":833,"exact":false,"location":"?","version":"?"},{"class":"com.ibm.watson.litelinks.ThreadPoolHelper$3$1","method":"run","file":"ThreadPoolHelper.java","line":91,"exact":false,"location":"litelinks-core-1.7.2.jar","version":"?"}]},"endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":22,"threadPriority":5} service failed, exiting {"instant":{"epochSecond":1669649041,"nanoOfSecond":723600998},"thread":"ll-svc-events-3","level":"INFO","loggerName":"com.ibm.watson.litelinks.server.ProbeHttpServer","message":"Stopping litelinks health probe http server on port 8089","endOfBatch":false,"loggerFqcn":"org.apache.logging.slf4j.Log4jLogger","contextMap":{},"threadId":20,"threadPriority":5} Wrote termination message to /dev/termination-log: "Service startup failed: Problem connecting to etcd" java.lang.IllegalStateException: Expected the service modelmesh-serving to be RUNNING, but the service has FAILED at com.google.common.util.concurrent.AbstractService.checkCurrentState(AbstractService.java:381) at com.google.common.util.concurrent.AbstractService.awaitRunning(AbstractService.java:321) at com.ibm.watson.litelinks.server.LitelinksService.run(LitelinksService.java:699) at com.ibm.watson.litelinks.server.LitelinksService.launch(LitelinksService.java:144) at com.ibm.watson.litelinks.server.LitelinksService.main(LitelinksService.java:108) Caused by: java.lang.Exception: Problem connecting to etcd at com.ibm.watson.modelmesh.ModelMesh.initialize(ModelMesh.java:617) at com.ibm.watson.litelinks.server.ThriftService.doStart(ThriftService.java:123) at com.google.common.util.concurrent.AbstractService.startAsync(AbstractService.java:251) at com.ibm.watson.litelinks.server.DefaultThriftServer.lambda$doStart$0(DefaultThriftServer.java:390) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:833) at com.ibm.watson.litelinks.ThreadPoolHelper$3$1.run(ThreadPoolHelper.java:91) Caused by: java.lang.RuntimeException: etcd connection verification failed at com.ibm.watson.etcd.EtcdUtilsFactory.lambda$verifyKvStoreConnection$1(EtcdUtilsFactory.java:172) at com.google.common.util.concurrent.AbstractCatchingFuture$CatchingFuture.doFallback(AbstractCatchingFuture.java:234) at com.google.common.util.concurrent.AbstractCatchingFuture$CatchingFuture.doFallback(AbstractCatchingFuture.java:222) at com.google.common.util.concurrent.AbstractCatchingFuture.run(AbstractCatchingFuture.java:133) at com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31) at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1270) at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038) at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:808) at com.google.common.util.concurrent.AbstractTransformFuture.run(AbstractTransformFuture.java:104) at com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31) at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1270) at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038) at com.google.common.util.concurrent.AbstractFuture.setFuture(AbstractFuture.java:851) at com.google.common.util.concurrent.AbstractCatchingFuture$AsyncCatchingFuture.setResult(AbstractCatchingFuture.java:214) at com.google.common.util.concurrent.AbstractCatchingFuture$AsyncCatchingFuture.setResult(AbstractCatchingFuture.java:190) at com.google.common.util.concurrent.AbstractCatchingFuture.run(AbstractCatchingFuture.java:142) at com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31) at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1270) at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038) at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:808) at io.grpc.stub.ClientCalls$GrpcFuture.setException(ClientCalls.java:574) at io.grpc.stub.ClientCalls$UnaryStreamToFuture.onClose(ClientCalls.java:544) at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:563) at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:70) at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:744) at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:723) at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:833) Caused by: io.grpc.StatusRuntimeException: UNAVAILABLE: io exception at io.grpc.Status.asRuntimeException(Status.java:539) ... 10 more Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: etcd.redhat-ods-applications/172.30.30.132:2379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at io.netty.channel.unix.Errors.newConnectException0(Errors.java:155) at io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:128) at io.netty.channel.unix.Socket.finishConnect(Socket.java:359) at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:710) at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:687) at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:833)