2024-06-20 10:45:02,406 ERROR [org.infinispan.PERSISTENCE] (blocking-thread--p3-t7) ISPN008025: Error while removing string keys from database: org.postgresql.util.PSQLException: ERROR: canceling statement due to lock timeout 2024-06-20T08:45:02.414164044Z Where: while deleting tuple (1025,4) in relation "ispn_clientSessions" 2024-06-20T08:45:02.414171418Z unnamed portal with parameters: $1 = 'af1a9775b-2fe1-4cfd-836f-8f8fb30759b2' 2024-06-20T08:45:02.414177548Z at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2725) 2024-06-20T08:45:02.414182217Z at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2412) 2024-06-20T08:45:02.414186715Z at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:371) 2024-06-20T08:45:02.414191811Z at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:502) 2024-06-20T08:45:02.414196596Z at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:419) 2024-06-20T08:45:02.414201065Z at org.postgresql.jdbc.PgPreparedStatement.executeWithFlags(PgPreparedStatement.java:194) 2024-06-20T08:45:02.414221012Z at org.postgresql.jdbc.PgPreparedStatement.executeUpdate(PgPreparedStatement.java:155) 2024-06-20T08:45:02.414226364Z at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 2024-06-20T08:45:02.414230961Z at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) 2024-06-20T08:45:02.414235542Z at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 2024-06-20T08:45:02.414240153Z at java.base/java.lang.reflect.Method.invoke(Method.java:568) 2024-06-20T08:45:02.414244926Z at org.postgresql.ds.PGPooledConnection$StatementHandler.invoke(PGPooledConnection.java:447) 2024-06-20T08:45:02.414249744Z at jdk.proxy2/jdk.proxy2.$Proxy85.executeUpdate(Unknown Source) 2024-06-20T08:45:02.414254851Z at io.agroal.pool.wrapper.PreparedStatementWrapper.executeUpdate(PreparedStatementWrapper.java:90) 2024-06-20T08:45:02.414259872Z at org.infinispan.persistence.jdbc.common.sql.BaseTableOperations.deleteEntry(BaseTableOperations.java:107) 2024-06-20T08:45:02.414265056Z at org.infinispan.persistence.jdbc.common.impl.BaseJdbcStore.lambda$delete$5(BaseJdbcStore.java:162) 2024-06-20T08:45:02.414272023Z at org.infinispan.util.concurrent.BlockingManagerImpl.supplyBlockingOperation(BlockingManagerImpl.java:145) 2024-06-20T08:45:02.414277216Z at org.infinispan.util.concurrent.BlockingManagerImpl.supplyBlocking(BlockingManagerImpl.java:136) 2024-06-20T08:45:02.414282438Z at org.infinispan.persistence.jdbc.common.impl.BaseJdbcStore.delete(BaseJdbcStore.java:158) 2024-06-20T08:45:02.414287324Z at org.infinispan.persistence.manager.PersistenceManagerImpl.deleteFromAllStores(PersistenceManagerImpl.java:740) 2024-06-20T08:45:02.414291997Z at org.infinispan.interceptors.impl.DistCacheWriterInterceptor.lambda$visitRemoveCommand$4(DistCacheWriterInterceptor.java:118) 2024-06-20T08:45:02.414296744Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenApply(BaseAsyncInterceptor.java:86) 2024-06-20T08:45:02.414301289Z at org.infinispan.interceptors.impl.DistCacheWriterInterceptor.visitRemoveCommand(DistCacheWriterInterceptor.java:111) 2024-06-20T08:45:02.414305981Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:02.414310340Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:02.414314878Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) 2024-06-20T08:45:02.414319466Z at org.infinispan.interceptors.BaseAsyncInterceptor.asyncInvokeNext(BaseAsyncInterceptor.java:232) 2024-06-20T08:45:02.414323987Z at org.infinispan.interceptors.impl.CacheLoaderInterceptor.visitDataCommand(CacheLoaderInterceptor.java:198) 2024-06-20T08:45:02.414328626Z at org.infinispan.interceptors.impl.CacheLoaderInterceptor.visitRemoveCommand(CacheLoaderInterceptor.java:157) 2024-06-20T08:45:02.414526869Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:02.414566502Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:02.414584285Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) 2024-06-20T08:45:02.414630270Z at org.infinispan.interceptors.BaseAsyncInterceptor.asyncInvokeNext(BaseAsyncInterceptor.java:232) 2024-06-20T08:45:02.414714308Z at org.infinispan.interceptors.impl.EntryWrappingInterceptor.setSkipRemoteGetsAndInvokeNextForDataCommand(EntryWrappingInterceptor.java:733) 2024-06-20T08:45:02.414733816Z at org.infinispan.interceptors.impl.EntryWrappingInterceptor.visitRemoveExpiredCommand(EntryWrappingInterceptor.java:414) 2024-06-20T08:45:02.414751971Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:02.414760312Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:155) 2024-06-20T08:45:02.414766336Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lambda$nonTxLockAndInvokeNext$3(AbstractLockingInterceptor.java:318) 2024-06-20T08:45:02.414771892Z at org.infinispan.interceptors.SyncInvocationStage.andHandle(SyncInvocationStage.java:69) 2024-06-20T08:45:02.414776568Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.nonTxLockAndInvokeNext(AbstractLockingInterceptor.java:313) 2024-06-20T08:45:02.414781264Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:138) 2024-06-20T08:45:02.414785921Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitRemoveExpiredCommand(AbstractLockingInterceptor.java:107) 2024-06-20T08:45:02.414790471Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:02.414795282Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndHandle(BaseAsyncInterceptor.java:188) 2024-06-20T08:45:02.414800220Z at org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:312) 2024-06-20T08:45:02.414805293Z at org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:256) 2024-06-20T08:45:02.414809734Z at org.infinispan.statetransfer.StateTransferInterceptor.visitRemoveCommand(StateTransferInterceptor.java:113) 2024-06-20T08:45:02.414814287Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:02.414818634Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:02.414823401Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:155) 2024-06-20T08:45:02.414827790Z at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitRemoveCommand(CacheMgmtInterceptor.java:507) 2024-06-20T08:45:02.414832336Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:02.414855108Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:02.414861264Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) 2024-06-20T08:45:02.414866173Z at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54) 2024-06-20T08:45:02.414872625Z at org.infinispan.interceptors.DDAsyncInterceptor.visitRemoveCommand(DDAsyncInterceptor.java:65) 2024-06-20T08:45:02.414877671Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:02.414882196Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:02.414886783Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:128) 2024-06-20T08:45:02.414891375Z at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90) 2024-06-20T08:45:02.414896015Z at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invokeAsync(AsyncInterceptorChainImpl.java:220) 2024-06-20T08:45:02.414901003Z at org.infinispan.cache.impl.InvocationHelper.doInvokeAsync(InvocationHelper.java:318) 2024-06-20T08:45:02.414906192Z at org.infinispan.cache.impl.InvocationHelper.invokeAsync(InvocationHelper.java:156) 2024-06-20T08:45:02.414911004Z at org.infinispan.cache.impl.InvocationHelper.invokeAsync(InvocationHelper.java:139) 2024-06-20T08:45:02.414915500Z at org.infinispan.cache.impl.CacheImpl.performVisitableNonTxCommand(CacheImpl.java:724) 2024-06-20T08:45:02.414920014Z at org.infinispan.cache.impl.CacheImpl.removeLifespanExpired(CacheImpl.java:706) 2024-06-20T08:45:02.414924543Z at org.infinispan.cache.impl.DecoratedCache.removeLifespanExpired(DecoratedCache.java:425) 2024-06-20T08:45:02.414929023Z at org.infinispan.expiration.impl.ClusterExpirationManager.handleInStoreExpirationInternal(ClusterExpirationManager.java:416) 2024-06-20T08:45:02.414933675Z at org.infinispan.expiration.impl.ClusterExpirationManager.handleInStoreExpirationInternal(ClusterExpirationManager.java:405) 2024-06-20T08:45:02.414938399Z at org.infinispan.persistence.manager.PersistenceManagerImpl.lambda$purgeExpired$28(PersistenceManagerImpl.java:674) 2024-06-20T08:45:02.414943155Z at io.reactivex.rxjava3.internal.operators.mixed.FlowableConcatMapCompletable$ConcatMapCompletableObserver.drain(FlowableConcatMapCompletable.java:187) 2024-06-20T08:45:02.414947993Z at io.reactivex.rxjava3.internal.operators.mixed.ConcatMapXMainSubscriber.onNext(ConcatMapXMainSubscriber.java:106) 2024-06-20T08:45:02.414952800Z at io.reactivex.rxjava3.processors.UnicastProcessor.drainFused(UnicastProcessor.java:349) 2024-06-20T08:45:02.414957817Z at io.reactivex.rxjava3.processors.UnicastProcessor.drain(UnicastProcessor.java:382) 2024-06-20T08:45:02.414962708Z at io.reactivex.rxjava3.processors.UnicastProcessor.onNext(UnicastProcessor.java:444) 2024-06-20T08:45:02.414974874Z at org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore.runBatchAndNotify(JdbcStringBasedStore.java:390) 2024-06-20T08:45:02.414980415Z at org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore.lambda$purgeExpired$1(JdbcStringBasedStore.java:348) 2024-06-20T08:45:02.414986428Z at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804) 2024-06-20T08:45:02.414991087Z at org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18) 2024-06-20T08:45:02.414995537Z at org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2513) 2024-06-20T08:45:02.414999996Z at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1538) 2024-06-20T08:45:02.415004504Z at java.base/java.lang.Thread.run(Thread.java:840) 2024-06-20T08:45:02.415008884Z 2024-06-20T08:45:10.750692022Z 2024-06-20 10:45:10,749 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (blocking-thread--p3-t7) ISPN000136: Error executing command RemoveExpiredCommand on Cache 'clientSessions', writing keys [f1a9775b-2fe1-4cfd-836f-8f8fb30759b2]: org.infinispan.persistence.spi.PersistenceException: Error while removing key f1a9775b-2fe1-4cfd-836f-8f8fb30759b2 from database 2024-06-20T08:45:10.750711735Z at org.infinispan.persistence.jdbc.common.impl.BaseJdbcStore.lambda$delete$5(BaseJdbcStore.java:165) 2024-06-20T08:45:10.750713505Z at org.infinispan.util.concurrent.BlockingManagerImpl.supplyBlockingOperation(BlockingManagerImpl.java:145) 2024-06-20T08:45:10.750714817Z at org.infinispan.util.concurrent.BlockingManagerImpl.supplyBlocking(BlockingManagerImpl.java:136) 2024-06-20T08:45:10.750716178Z at org.infinispan.persistence.jdbc.common.impl.BaseJdbcStore.delete(BaseJdbcStore.java:158) 2024-06-20T08:45:10.750717483Z at org.infinispan.persistence.manager.PersistenceManagerImpl.deleteFromAllStores(PersistenceManagerImpl.java:740) 2024-06-20T08:45:10.750718800Z at org.infinispan.interceptors.impl.DistCacheWriterInterceptor.lambda$visitRemoveCommand$4(DistCacheWriterInterceptor.java:118) 2024-06-20T08:45:10.750720157Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenApply(BaseAsyncInterceptor.java:86) 2024-06-20T08:45:10.750721511Z at org.infinispan.interceptors.impl.DistCacheWriterInterceptor.visitRemoveCommand(DistCacheWriterInterceptor.java:111) 2024-06-20T08:45:10.750722905Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:10.750724165Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:10.750725685Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) 2024-06-20T08:45:10.750731936Z at org.infinispan.interceptors.BaseAsyncInterceptor.asyncInvokeNext(BaseAsyncInterceptor.java:232) 2024-06-20T08:45:10.750732927Z at org.infinispan.interceptors.impl.CacheLoaderInterceptor.visitDataCommand(CacheLoaderInterceptor.java:198) 2024-06-20T08:45:10.750734418Z at org.infinispan.interceptors.impl.CacheLoaderInterceptor.visitRemoveCommand(CacheLoaderInterceptor.java:157) 2024-06-20T08:45:10.750735405Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:10.750736325Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:10.750737292Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) 2024-06-20T08:45:10.750738226Z at org.infinispan.interceptors.BaseAsyncInterceptor.asyncInvokeNext(BaseAsyncInterceptor.java:232) 2024-06-20T08:45:10.750739227Z at org.infinispan.interceptors.impl.EntryWrappingInterceptor.setSkipRemoteGetsAndInvokeNextForDataCommand(EntryWrappingInterceptor.java:733) 2024-06-20T08:45:10.750740236Z at org.infinispan.interceptors.impl.EntryWrappingInterceptor.visitRemoveExpiredCommand(EntryWrappingInterceptor.java:414) 2024-06-20T08:45:10.750741209Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:10.750742153Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:155) 2024-06-20T08:45:10.750743100Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lambda$nonTxLockAndInvokeNext$3(AbstractLockingInterceptor.java:318) 2024-06-20T08:45:10.750744123Z at org.infinispan.interceptors.SyncInvocationStage.andHandle(SyncInvocationStage.java:69) 2024-06-20T08:45:10.750745094Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.nonTxLockAndInvokeNext(AbstractLockingInterceptor.java:313) 2024-06-20T08:45:10.750746068Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:138) 2024-06-20T08:45:10.750747080Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitRemoveExpiredCommand(AbstractLockingInterceptor.java:107) 2024-06-20T08:45:10.750748085Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:10.750749024Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndHandle(BaseAsyncInterceptor.java:188) 2024-06-20T08:45:10.750749955Z at org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:312) 2024-06-20T08:45:10.750750901Z at org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:256) 2024-06-20T08:45:10.750751892Z at org.infinispan.statetransfer.StateTransferInterceptor.visitRemoveCommand(StateTransferInterceptor.java:113) 2024-06-20T08:45:10.750753058Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:10.750753998Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:10.750756316Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:155) 2024-06-20T08:45:10.750757300Z at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitRemoveCommand(CacheMgmtInterceptor.java:507) 2024-06-20T08:45:10.750758283Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:10.750759194Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:10.750760139Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) 2024-06-20T08:45:10.750761085Z at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54) 2024-06-20T08:45:10.750762074Z at org.infinispan.interceptors.DDAsyncInterceptor.visitRemoveCommand(DDAsyncInterceptor.java:65) 2024-06-20T08:45:10.750763016Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:10.750763935Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:10.750764867Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:128) 2024-06-20T08:45:10.750765813Z at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90) 2024-06-20T08:45:10.750766771Z at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invokeAsync(AsyncInterceptorChainImpl.java:220) 2024-06-20T08:45:10.750767723Z at org.infinispan.cache.impl.InvocationHelper.doInvokeAsync(InvocationHelper.java:318) 2024-06-20T08:45:10.750768652Z at org.infinispan.cache.impl.InvocationHelper.invokeAsync(InvocationHelper.java:156) 2024-06-20T08:45:10.750769612Z at org.infinispan.cache.impl.InvocationHelper.invokeAsync(InvocationHelper.java:139) 2024-06-20T08:45:10.750770751Z at org.infinispan.cache.impl.CacheImpl.performVisitableNonTxCommand(CacheImpl.java:724) 2024-06-20T08:45:10.750771688Z at org.infinispan.cache.impl.CacheImpl.removeLifespanExpired(CacheImpl.java:706) 2024-06-20T08:45:10.750772595Z at org.infinispan.cache.impl.DecoratedCache.removeLifespanExpired(DecoratedCache.java:425) 2024-06-20T08:45:10.750773533Z at org.infinispan.expiration.impl.ClusterExpirationManager.handleInStoreExpirationInternal(ClusterExpirationManager.java:416) 2024-06-20T08:45:10.750774652Z at org.infinispan.expiration.impl.ClusterExpirationManager.handleInStoreExpirationInternal(ClusterExpirationManager.java:405) 2024-06-20T08:45:10.750775682Z at org.infinispan.persistence.manager.PersistenceManagerImpl.lambda$purgeExpired$28(PersistenceManagerImpl.java:674) 2024-06-20T08:45:10.750776660Z at io.reactivex.rxjava3.internal.operators.mixed.FlowableConcatMapCompletable$ConcatMapCompletableObserver.drain(FlowableConcatMapCompletable.java:187) 2024-06-20T08:45:10.750777712Z at io.reactivex.rxjava3.internal.operators.mixed.ConcatMapXMainSubscriber.onNext(ConcatMapXMainSubscriber.java:106) 2024-06-20T08:45:10.750779725Z at io.reactivex.rxjava3.processors.UnicastProcessor.drainFused(UnicastProcessor.java:349) 2024-06-20T08:45:10.750780721Z at io.reactivex.rxjava3.processors.UnicastProcessor.drain(UnicastProcessor.java:382) 2024-06-20T08:45:10.750781691Z at io.reactivex.rxjava3.processors.UnicastProcessor.onNext(UnicastProcessor.java:444) 2024-06-20T08:45:10.750782619Z at org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore.runBatchAndNotify(JdbcStringBasedStore.java:390) 2024-06-20T08:45:10.750783585Z at org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore.lambda$purgeExpired$1(JdbcStringBasedStore.java:348) 2024-06-20T08:45:10.750784569Z at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804) 2024-06-20T08:45:10.750785577Z at org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18) 2024-06-20T08:45:10.750786615Z at org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2513) 2024-06-20T08:45:10.750787610Z at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1538) 2024-06-20T08:45:10.750788702Z at java.base/java.lang.Thread.run(Thread.java:840) 2024-06-20T08:45:10.750789702Z Caused by: org.postgresql.util.PSQLException: ERROR: canceling statement due to lock timeout 2024-06-20T08:45:10.750790676Z Where: while deleting tuple (1025,4) in relation "ispn_clientSessions" 2024-06-20T08:45:10.750791746Z unnamed portal with parameters: $1 = 'af1a9775b-2fe1-4cfd-836f-8f8fb30759b2' 2024-06-20T08:45:10.750793098Z at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2725) 2024-06-20T08:45:10.750794062Z at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2412) 2024-06-20T08:45:10.750795088Z at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:371) 2024-06-20T08:45:10.750796034Z at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:502) 2024-06-20T08:45:10.750796955Z at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:419) 2024-06-20T08:45:10.750799144Z at org.postgresql.jdbc.PgPreparedStatement.executeWithFlags(PgPreparedStatement.java:194) 2024-06-20T08:45:10.750800105Z at org.postgresql.jdbc.PgPreparedStatement.executeUpdate(PgPreparedStatement.java:155) 2024-06-20T08:45:10.750801051Z at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 2024-06-20T08:45:10.750802060Z at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) 2024-06-20T08:45:10.750803074Z at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 2024-06-20T08:45:10.750804122Z at java.base/java.lang.reflect.Method.invoke(Method.java:568) 2024-06-20T08:45:10.750805027Z at org.postgresql.ds.PGPooledConnection$StatementHandler.invoke(PGPooledConnection.java:447) 2024-06-20T08:45:10.750807172Z at jdk.proxy2/jdk.proxy2.$Proxy85.executeUpdate(Unknown Source) 2024-06-20T08:45:10.750808120Z at io.agroal.pool.wrapper.PreparedStatementWrapper.executeUpdate(PreparedStatementWrapper.java:90) 2024-06-20T08:45:10.750809107Z at org.infinispan.persistence.jdbc.common.sql.BaseTableOperations.deleteEntry(BaseTableOperations.java:107) 2024-06-20T08:45:10.750810062Z at org.infinispan.persistence.jdbc.common.impl.BaseJdbcStore.lambda$delete$5(BaseJdbcStore.java:162) 2024-06-20T08:45:10.750811018Z ... 66 more 2024-06-20T08:45:10.750811920Z 2024-06-20T08:45:11.206464550Z 2024-06-20 10:45:11,205 ERROR [org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore] (blocking-thread--p3-t7) ISPN008001: Failed clearing cache store: java.sql.BatchUpdateException: Batch entry 0 DELETE FROM "ispn_clientSessions" WHERE id = ('addbc1d25-cdee-4be0-b123-2ca28d0961f5') AND ts = ('1718763715000'::int8) was aborted: An I/O error occurred while sending to the backend. Call getNextException to see other errors in the batch. 2024-06-20T08:45:11.206490419Z at org.postgresql.jdbc.BatchResultHandler.handleError(BatchResultHandler.java:165) 2024-06-20T08:45:11.206495483Z at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:584) 2024-06-20T08:45:11.206498907Z at org.postgresql.jdbc.PgStatement.internalExecuteBatch(PgStatement.java:912) 2024-06-20T08:45:11.206501709Z at org.postgresql.jdbc.PgStatement.executeBatch(PgStatement.java:936) 2024-06-20T08:45:11.206504429Z at org.postgresql.jdbc.PgPreparedStatement.executeBatch(PgPreparedStatement.java:1733) 2024-06-20T08:45:11.206507145Z at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 2024-06-20T08:45:11.206510080Z at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) 2024-06-20T08:45:11.206512936Z at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 2024-06-20T08:45:11.206515856Z at java.base/java.lang.reflect.Method.invoke(Method.java:568) 2024-06-20T08:45:11.206518494Z at org.postgresql.ds.PGPooledConnection$StatementHandler.invoke(PGPooledConnection.java:447) 2024-06-20T08:45:11.206521301Z at jdk.proxy2/jdk.proxy2.$Proxy85.executeBatch(Unknown Source) 2024-06-20T08:45:11.206524085Z at io.agroal.pool.wrapper.StatementWrapper.executeBatch(StatementWrapper.java:340) 2024-06-20T08:45:11.206526786Z at org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore.runBatchAndNotify(JdbcStringBasedStore.java:384) 2024-06-20T08:45:11.206529889Z at org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore.lambda$purgeExpired$1(JdbcStringBasedStore.java:348) 2024-06-20T08:45:11.206532963Z at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804) 2024-06-20T08:45:11.206546158Z at org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18) 2024-06-20T08:45:11.206550643Z at org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2513) 2024-06-20T08:45:11.206553776Z at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1538) 2024-06-20T08:45:11.206556596Z at java.base/java.lang.Thread.run(Thread.java:840) 2024-06-20T08:45:11.206559246Z Caused by: org.postgresql.util.PSQLException: An I/O error occurred while sending to the backend. 2024-06-20T08:45:11.206562021Z at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:585) 2024-06-20T08:45:11.206564721Z ... 17 more 2024-06-20T08:45:11.206568192Z Caused by: java.net.SocketException: Broken pipe 2024-06-20T08:45:11.206571469Z at java.base/sun.nio.ch.NioSocketImpl.implWrite(NioSocketImpl.java:425) 2024-06-20T08:45:11.206574642Z at java.base/sun.nio.ch.NioSocketImpl.write(NioSocketImpl.java:445) 2024-06-20T08:45:11.206577740Z at java.base/sun.nio.ch.NioSocketImpl$2.write(NioSocketImpl.java:831) 2024-06-20T08:45:11.206580837Z at java.base/java.net.Socket$SocketOutputStream.write(Socket.java:1035) 2024-06-20T08:45:11.206583985Z at java.base/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) 2024-06-20T08:45:11.206587120Z at java.base/java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) 2024-06-20T08:45:11.206590262Z at org.postgresql.core.PGStream.flush(PGStream.java:724) 2024-06-20T08:45:11.206593334Z at org.postgresql.core.v3.QueryExecutorImpl.sendSync(QueryExecutorImpl.java:1574) 2024-06-20T08:45:11.206596483Z at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:577) 2024-06-20T08:45:11.206599680Z ... 17 more 2024-06-20T08:45:11.206602761Z 2024-06-20T08:45:12.867044649Z 2024-06-20 10:45:12,866 ERROR [org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore] (blocking-thread--p3-t7) ISPN008032: Error during rollback of JDBC transaction: org.postgresql.util.PSQLException: Connection has been closed. 2024-06-20T08:45:12.870439535Z at org.postgresql.ds.PGPooledConnection$ConnectionHandler.invoke(PGPooledConnection.java:332) 2024-06-20T08:45:12.870461427Z at jdk.proxy2/jdk.proxy2.$Proxy56.rollback(Unknown Source) 2024-06-20T08:45:12.870465833Z at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 2024-06-20T08:45:12.870469140Z at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) 2024-06-20T08:45:12.870472490Z at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 2024-06-20T08:45:12.870492124Z at java.base/java.lang.reflect.Method.invoke(Method.java:568) 2024-06-20T08:45:12.870495745Z at org.postgresql.xa.PGXAConnection$ConnectionHandler.invoke(PGXAConnection.java:153) 2024-06-20T08:45:12.870498857Z at jdk.proxy2/jdk.proxy2.$Proxy56.rollback(Unknown Source) 2024-06-20T08:45:12.870501847Z at io.agroal.pool.wrapper.ConnectionWrapper.rollback(ConnectionWrapper.java:208) 2024-06-20T08:45:12.870504872Z at org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore.lambda$purgeExpired$1(JdbcStringBasedStore.java:366) 2024-06-20T08:45:12.870508337Z at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804) 2024-06-20T08:45:12.870511384Z at org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18) 2024-06-20T08:45:12.870514333Z at org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2513) 2024-06-20T08:45:12.870517473Z at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1538) 2024-06-20T08:45:12.870520597Z at java.base/java.lang.Thread.run(Thread.java:840) 2024-06-20T08:45:12.870523602Z 2024-06-20T08:45:43.371452561Z 2024-06-20 10:45:43,370 WARN [io.agroal.pool] (blocking-thread--p3-t7) Datasource 'infinispan-db': Connection has been closed. 2024-06-20T08:45:46.394813692Z 2024-06-20 10:45:46,393 ERROR [org.infinispan.PERSISTENCE] (blocking-thread--p3-t4) ISPN008025: Error while removing string keys from database: org.postgresql.util.PSQLException: ERROR: canceling statement due to lock timeout 2024-06-20T08:45:46.394851570Z Where: while deleting tuple (2236,2) in relation "ispn_sessions" 2024-06-20T08:45:46.394860837Z unnamed portal with parameters: $1 = 'f8204567-6be1-4e0d-b57d-100650fecdf3' 2024-06-20T08:45:46.394868187Z at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2725) 2024-06-20T08:45:46.394875356Z at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2412) 2024-06-20T08:45:46.394882218Z at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:371) 2024-06-20T08:45:46.394888907Z at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:502) 2024-06-20T08:45:46.394895969Z at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:419) 2024-06-20T08:45:46.394902667Z at org.postgresql.jdbc.PgPreparedStatement.executeWithFlags(PgPreparedStatement.java:194) 2024-06-20T08:45:46.394909520Z at org.postgresql.jdbc.PgPreparedStatement.executeUpdate(PgPreparedStatement.java:155) 2024-06-20T08:45:46.394933934Z at jdk.internal.reflect.GeneratedMethodAccessor24.invoke(Unknown Source) 2024-06-20T08:45:46.394939998Z at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 2024-06-20T08:45:46.394945207Z at java.base/java.lang.reflect.Method.invoke(Method.java:568) 2024-06-20T08:45:46.394950126Z at org.postgresql.ds.PGPooledConnection$StatementHandler.invoke(PGPooledConnection.java:447) 2024-06-20T08:45:46.394955123Z at jdk.proxy2/jdk.proxy2.$Proxy85.executeUpdate(Unknown Source) 2024-06-20T08:45:46.394960069Z at io.agroal.pool.wrapper.PreparedStatementWrapper.executeUpdate(PreparedStatementWrapper.java:90) 2024-06-20T08:45:46.394965043Z at org.infinispan.persistence.jdbc.common.sql.BaseTableOperations.deleteEntry(BaseTableOperations.java:107) 2024-06-20T08:45:46.394970403Z at org.infinispan.persistence.jdbc.common.impl.BaseJdbcStore.lambda$delete$5(BaseJdbcStore.java:162) 2024-06-20T08:45:46.394975191Z at org.infinispan.util.concurrent.BlockingManagerImpl.supplyBlockingOperation(BlockingManagerImpl.java:145) 2024-06-20T08:45:46.394980383Z at org.infinispan.util.concurrent.BlockingManagerImpl.supplyBlocking(BlockingManagerImpl.java:136) 2024-06-20T08:45:46.394987294Z at org.infinispan.persistence.jdbc.common.impl.BaseJdbcStore.delete(BaseJdbcStore.java:158) 2024-06-20T08:45:46.394992352Z at org.infinispan.persistence.manager.PersistenceManagerImpl.deleteFromAllStores(PersistenceManagerImpl.java:740) 2024-06-20T08:45:46.394997493Z at org.infinispan.interceptors.impl.DistCacheWriterInterceptor.lambda$visitRemoveCommand$4(DistCacheWriterInterceptor.java:118) 2024-06-20T08:45:46.395002777Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenApply(BaseAsyncInterceptor.java:86) 2024-06-20T08:45:46.395007654Z at org.infinispan.interceptors.impl.DistCacheWriterInterceptor.visitRemoveCommand(DistCacheWriterInterceptor.java:111) 2024-06-20T08:45:46.395012680Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:46.395017530Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:46.395022139Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) 2024-06-20T08:45:46.395026788Z at org.infinispan.interceptors.BaseAsyncInterceptor.asyncInvokeNext(BaseAsyncInterceptor.java:232) 2024-06-20T08:45:46.395031559Z at org.infinispan.interceptors.impl.CacheLoaderInterceptor.visitDataCommand(CacheLoaderInterceptor.java:198) 2024-06-20T08:45:46.395036225Z at org.infinispan.interceptors.impl.CacheLoaderInterceptor.visitRemoveCommand(CacheLoaderInterceptor.java:157) 2024-06-20T08:45:46.395041678Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:46.395047228Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:46.395052114Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) 2024-06-20T08:45:46.395063992Z at org.infinispan.interceptors.BaseAsyncInterceptor.asyncInvokeNext(BaseAsyncInterceptor.java:232) 2024-06-20T08:45:46.395069282Z at org.infinispan.interceptors.impl.EntryWrappingInterceptor.setSkipRemoteGetsAndInvokeNextForDataCommand(EntryWrappingInterceptor.java:733) 2024-06-20T08:45:46.395074246Z at org.infinispan.interceptors.impl.EntryWrappingInterceptor.visitRemoveExpiredCommand(EntryWrappingInterceptor.java:414) 2024-06-20T08:45:46.395078985Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:46.395083664Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:155) 2024-06-20T08:45:46.395089506Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lambda$nonTxLockAndInvokeNext$3(AbstractLockingInterceptor.java:318) 2024-06-20T08:45:46.395094442Z at org.infinispan.interceptors.SyncInvocationStage.andHandle(SyncInvocationStage.java:69) 2024-06-20T08:45:46.395099006Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.nonTxLockAndInvokeNext(AbstractLockingInterceptor.java:313) 2024-06-20T08:45:46.395103847Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:138) 2024-06-20T08:45:46.395108671Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitRemoveExpiredCommand(AbstractLockingInterceptor.java:107) 2024-06-20T08:45:46.395113364Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:46.395118090Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndHandle(BaseAsyncInterceptor.java:188) 2024-06-20T08:45:46.395122871Z at org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:312) 2024-06-20T08:45:46.395127793Z at org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:256) 2024-06-20T08:45:46.395132469Z at org.infinispan.statetransfer.StateTransferInterceptor.visitRemoveCommand(StateTransferInterceptor.java:113) 2024-06-20T08:45:46.395137627Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:46.395142500Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:46.395147383Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:155) 2024-06-20T08:45:46.395152146Z at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitRemoveCommand(CacheMgmtInterceptor.java:507) 2024-06-20T08:45:46.395157377Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:46.395162584Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:46.395167729Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) 2024-06-20T08:45:46.395178595Z at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54) 2024-06-20T08:45:46.395184809Z at org.infinispan.interceptors.DDAsyncInterceptor.visitRemoveCommand(DDAsyncInterceptor.java:65) 2024-06-20T08:45:46.395189674Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:45:46.395197786Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:45:46.395203640Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:128) 2024-06-20T08:45:46.395208855Z at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90) 2024-06-20T08:45:46.395213580Z at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invokeAsync(AsyncInterceptorChainImpl.java:220) 2024-06-20T08:45:46.395218764Z at org.infinispan.cache.impl.InvocationHelper.doInvokeAsync(InvocationHelper.java:318) 2024-06-20T08:45:46.395223488Z at org.infinispan.cache.impl.InvocationHelper.invokeAsync(InvocationHelper.java:156) 2024-06-20T08:45:46.395228225Z at org.infinispan.cache.impl.InvocationHelper.invokeAsync(InvocationHelper.java:139) 2024-06-20T08:45:46.395233227Z at org.infinispan.cache.impl.CacheImpl.performVisitableNonTxCommand(CacheImpl.java:724) 2024-06-20T08:45:46.395237856Z at org.infinispan.cache.impl.CacheImpl.removeLifespanExpired(CacheImpl.java:706) 2024-06-20T08:45:46.395242538Z at org.infinispan.cache.impl.DecoratedCache.removeLifespanExpired(DecoratedCache.java:425) 2024-06-20T08:45:46.395247404Z at org.infinispan.expiration.impl.ClusterExpirationManager.handleInStoreExpirationInternal(ClusterExpirationManager.java:416) 2024-06-20T08:45:46.395252207Z at org.infinispan.expiration.impl.ClusterExpirationManager.handleInStoreExpirationInternal(ClusterExpirationManager.java:405) 2024-06-20T08:45:46.395257161Z at org.infinispan.persistence.manager.PersistenceManagerImpl.lambda$purgeExpired$28(PersistenceManagerImpl.java:674) 2024-06-20T08:45:46.395262247Z at io.reactivex.rxjava3.internal.operators.mixed.FlowableConcatMapCompletable$ConcatMapCompletableObserver.drain(FlowableConcatMapCompletable.java:187) 2024-06-20T08:45:46.395267356Z at io.reactivex.rxjava3.internal.operators.mixed.ConcatMapXMainSubscriber.onNext(ConcatMapXMainSubscriber.java:106) 2024-06-20T08:45:46.395272118Z at io.reactivex.rxjava3.processors.UnicastProcessor.drainFused(UnicastProcessor.java:349) 2024-06-20T08:45:46.395276711Z at io.reactivex.rxjava3.processors.UnicastProcessor.drain(UnicastProcessor.java:382) 2024-06-20T08:45:46.395281313Z at io.reactivex.rxjava3.processors.UnicastProcessor.onNext(UnicastProcessor.java:444) 2024-06-20T08:45:46.395286250Z at org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore.runBatchAndNotify(JdbcStringBasedStore.java:390) 2024-06-20T08:45:46.395292907Z at org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore.lambda$purgeExpired$1(JdbcStringBasedStore.java:348) 2024-06-20T08:45:46.395305326Z at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804) 2024-06-20T08:45:46.395310818Z at org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18) 2024-06-20T08:45:46.395316465Z at org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2513) 2024-06-20T08:45:46.395321141Z at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1538) 2024-06-20T08:45:46.395325780Z at java.base/java.lang.Thread.run(Thread.java:840) 2024-06-20T08:45:46.395330528Z 2024-06-20T08:48:32.585969078Z 2024-06-20 10:48:32,577 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (blocking-thread--p3-t4) ISPN000136: Error executing command RemoveExpiredCommand on Cache 'sessions', writing keys [f8204567-6be1-4e0d-b57d-100650fecdf3]: org.infinispan.persistence.spi.PersistenceException: Error while removing key f8204567-6be1-4e0d-b57d-100650fecdf3 from database 2024-06-20T08:48:32.585999924Z at org.infinispan.persistence.jdbc.common.impl.BaseJdbcStore.lambda$delete$5(BaseJdbcStore.java:165) 2024-06-20T08:48:32.586002023Z at org.infinispan.util.concurrent.BlockingManagerImpl.supplyBlockingOperation(BlockingManagerImpl.java:145) 2024-06-20T08:48:32.586003500Z at org.infinispan.util.concurrent.BlockingManagerImpl.supplyBlocking(BlockingManagerImpl.java:136) 2024-06-20T08:48:32.586005001Z at org.infinispan.persistence.jdbc.common.impl.BaseJdbcStore.delete(BaseJdbcStore.java:158) 2024-06-20T08:48:32.586006635Z at org.infinispan.persistence.manager.PersistenceManagerImpl.deleteFromAllStores(PersistenceManagerImpl.java:740) 2024-06-20T08:48:32.586008013Z at org.infinispan.interceptors.impl.DistCacheWriterInterceptor.lambda$visitRemoveCommand$4(DistCacheWriterInterceptor.java:118) 2024-06-20T08:48:32.586009318Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenApply(BaseAsyncInterceptor.java:86) 2024-06-20T08:48:32.586010599Z at org.infinispan.interceptors.impl.DistCacheWriterInterceptor.visitRemoveCommand(DistCacheWriterInterceptor.java:111) 2024-06-20T08:48:32.586011970Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:48:32.586013288Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:48:32.586014588Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) 2024-06-20T08:48:32.586015840Z at org.infinispan.interceptors.BaseAsyncInterceptor.asyncInvokeNext(BaseAsyncInterceptor.java:232) 2024-06-20T08:48:32.586017129Z at org.infinispan.interceptors.impl.CacheLoaderInterceptor.visitDataCommand(CacheLoaderInterceptor.java:198) 2024-06-20T08:48:32.586019120Z at org.infinispan.interceptors.impl.CacheLoaderInterceptor.visitRemoveCommand(CacheLoaderInterceptor.java:157) 2024-06-20T08:48:32.586028502Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:48:32.586029469Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:48:32.586030484Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) 2024-06-20T08:48:32.586031443Z at org.infinispan.interceptors.BaseAsyncInterceptor.asyncInvokeNext(BaseAsyncInterceptor.java:232) 2024-06-20T08:48:32.586032416Z at org.infinispan.interceptors.impl.EntryWrappingInterceptor.setSkipRemoteGetsAndInvokeNextForDataCommand(EntryWrappingInterceptor.java:733) 2024-06-20T08:48:32.586033464Z at org.infinispan.interceptors.impl.EntryWrappingInterceptor.visitRemoveExpiredCommand(EntryWrappingInterceptor.java:414) 2024-06-20T08:48:32.586034461Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:48:32.586035402Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:155) 2024-06-20T08:48:32.586036369Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lambda$nonTxLockAndInvokeNext$3(AbstractLockingInterceptor.java:318) 2024-06-20T08:48:32.586037387Z at org.infinispan.interceptors.SyncInvocationStage.andHandle(SyncInvocationStage.java:69) 2024-06-20T08:48:32.586038349Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.nonTxLockAndInvokeNext(AbstractLockingInterceptor.java:313) 2024-06-20T08:48:32.586039333Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:138) 2024-06-20T08:48:32.586040341Z at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitRemoveExpiredCommand(AbstractLockingInterceptor.java:107) 2024-06-20T08:48:32.586041350Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:48:32.586042302Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndHandle(BaseAsyncInterceptor.java:188) 2024-06-20T08:48:32.586043276Z at org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:312) 2024-06-20T08:48:32.586044249Z at org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:256) 2024-06-20T08:48:32.586045197Z at org.infinispan.statetransfer.StateTransferInterceptor.visitRemoveCommand(StateTransferInterceptor.java:113) 2024-06-20T08:48:32.586046379Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:48:32.586047364Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:48:32.586048308Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:155) 2024-06-20T08:48:32.586049238Z at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitRemoveCommand(CacheMgmtInterceptor.java:507) 2024-06-20T08:48:32.586051560Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:48:32.586052508Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:48:32.586053462Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) 2024-06-20T08:48:32.586054395Z at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54) 2024-06-20T08:48:32.586055376Z at org.infinispan.interceptors.DDAsyncInterceptor.visitRemoveCommand(DDAsyncInterceptor.java:65) 2024-06-20T08:48:32.586056315Z at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:65) 2024-06-20T08:48:32.586057237Z at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:47) 2024-06-20T08:48:32.586058204Z at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:128) 2024-06-20T08:48:32.586059146Z at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90) 2024-06-20T08:48:32.586060112Z at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invokeAsync(AsyncInterceptorChainImpl.java:220) 2024-06-20T08:48:32.586061088Z at org.infinispan.cache.impl.InvocationHelper.doInvokeAsync(InvocationHelper.java:318) 2024-06-20T08:48:32.586062027Z at org.infinispan.cache.impl.InvocationHelper.invokeAsync(InvocationHelper.java:156) 2024-06-20T08:48:32.586062974Z at org.infinispan.cache.impl.InvocationHelper.invokeAsync(InvocationHelper.java:139) 2024-06-20T08:48:32.586063905Z at org.infinispan.cache.impl.CacheImpl.performVisitableNonTxCommand(CacheImpl.java:724) 2024-06-20T08:48:32.586064844Z at org.infinispan.cache.impl.CacheImpl.removeLifespanExpired(CacheImpl.java:706) 2024-06-20T08:48:32.586065807Z at org.infinispan.cache.impl.DecoratedCache.removeLifespanExpired(DecoratedCache.java:425) 2024-06-20T08:48:32.586066756Z at org.infinispan.expiration.impl.ClusterExpirationManager.handleInStoreExpirationInternal(ClusterExpirationManager.java:416) 2024-06-20T08:48:32.586068583Z at org.infinispan.expiration.impl.ClusterExpirationManager.handleInStoreExpirationInternal(ClusterExpirationManager.java:405) 2024-06-20T08:48:32.586069673Z at org.infinispan.persistence.manager.PersistenceManagerImpl.lambda$purgeExpired$28(PersistenceManagerImpl.java:674) 2024-06-20T08:48:32.586070651Z at io.reactivex.rxjava3.internal.operators.mixed.FlowableConcatMapCompletable$ConcatMapCompletableObserver.drain(FlowableConcatMapCompletable.java:187) 2024-06-20T08:48:32.586071689Z at io.reactivex.rxjava3.internal.operators.mixed.ConcatMapXMainSubscriber.onNext(ConcatMapXMainSubscriber.java:106) 2024-06-20T08:48:32.586072676Z at io.reactivex.rxjava3.processors.UnicastProcessor.drainFused(UnicastProcessor.java:349) 2024-06-20T08:48:32.586073609Z at io.reactivex.rxjava3.processors.UnicastProcessor.drain(UnicastProcessor.java:382) 2024-06-20T08:48:32.586074531Z at io.reactivex.rxjava3.processors.UnicastProcessor.onNext(UnicastProcessor.java:444) 2024-06-20T08:48:32.586076395Z at org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore.runBatchAndNotify(JdbcStringBasedStore.java:390) 2024-06-20T08:48:32.586077375Z at org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore.lambda$purgeExpired$1(JdbcStringBasedStore.java:348) 2024-06-20T08:48:32.586078393Z at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804) 2024-06-20T08:48:32.586079429Z at org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18) 2024-06-20T08:48:32.586080550Z at org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2513) 2024-06-20T08:48:32.586081503Z at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1538) 2024-06-20T08:48:32.586082447Z at java.base/java.lang.Thread.run(Thread.java:840) 2024-06-20T08:48:32.586083583Z Caused by: org.postgresql.util.PSQLException: ERROR: canceling statement due to lock timeout 2024-06-20T08:48:32.586084522Z Where: while deleting tuple (2236,2) in relation "ispn_sessions" 2024-06-20T08:48:32.586085552Z unnamed portal with parameters: $1 = 'f8204567-6be1-4e0d-b57d-100650fecdf3' 2024-06-20T08:48:32.586086447Z at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2725) 2024-06-20T08:48:32.586087378Z at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2412) 2024-06-20T08:48:32.586088332Z at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:371) 2024-06-20T08:48:32.586089270Z at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:502) 2024-06-20T08:48:32.586090183Z at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:419) 2024-06-20T08:48:32.586091314Z at org.postgresql.jdbc.PgPreparedStatement.executeWithFlags(PgPreparedStatement.java:194) 2024-06-20T08:48:32.586092253Z at org.postgresql.jdbc.PgPreparedStatement.executeUpdate(PgPreparedStatement.java:155) 2024-06-20T08:48:32.586093178Z at jdk.internal.reflect.GeneratedMethodAccessor24.invoke(Unknown Source) 2024-06-20T08:48:32.586094100Z at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 2024-06-20T08:48:32.586095081Z at java.base/java.lang.reflect.Method.invoke(Method.java:568) 2024-06-20T08:48:32.586096044Z at org.postgresql.ds.PGPooledConnection$StatementHandler.invoke(PGPooledConnection.java:447) 2024-06-20T08:48:32.586096982Z at jdk.proxy2/jdk.proxy2.$Proxy85.executeUpdate(Unknown Source) 2024-06-20T08:48:32.586097903Z at io.agroal.pool.wrapper.PreparedStatementWrapper.executeUpdate(PreparedStatementWrapper.java:90) 2024-06-20T08:48:32.586098845Z at org.infinispan.persistence.jdbc.common.sql.BaseTableOperations.deleteEntry(BaseTableOperations.java:107) 2024-06-20T08:48:32.586099808Z at org.infinispan.persistence.jdbc.common.impl.BaseJdbcStore.lambda$delete$5(BaseJdbcStore.java:162) 2024-06-20T08:48:32.586101781Z ... 66 more