Loading...

Type: Bug
Resolution: Done
Priority: Critical
Fix Version/s: 13.0.3.Final, 14.0.0.Final
Affects Version/s: None
Component/s: Operator
Labels:
None

Steps to Reproduce:

Hide

Follow up steps of ~~JDG-5031~~, shutdown the cluster by setting replicas to 0 then bring it back up.

Show
Follow up steps of JDG-5031 , shutdown the cluster by setting replicas to 0 then bring it back up.
Git Pull Request:
https://github.com/infinispan/infinispan/pull/9732, https://github.com/infinispan/infinispan/pull/9733

Scaling cluster with persisted entries 3->1->3->0->3 will result in bunch of WARN being generated in node-1, possibly making cache inaccesible.

12:49:22,564 WARN  (jgroups-12,infinispan-1-45756) [org.infinispan.CLUSTER] ISPN000071: Caught exception when handling command org.infinispan.manager.impl.ReplicableManagerFunctionCommand@5c2bb4a8 org.infinispan.commons.CacheConfigurationException: Error starting component org.infinispan.persistence.manager.PersistenceManager
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:585)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:808)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:635)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:599)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:577)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:808)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:635)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:599)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:577)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:808)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:635)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:599)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:577)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:808)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:635)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:599)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:577)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:808)
	at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:354)
	at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:250)
	at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:216)
	at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1014)
	at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:512)
	at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:700)
	at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:646)
	at org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:535)
	at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:513)
	at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:26)
	at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:14)
	at org.infinispan.security.Security.doPrivileged(Security.java:56)
	at org.infinispan.globalstate.impl.SecurityActions.doPrivileged(SecurityActions.java:30)
	at org.infinispan.globalstate.impl.SecurityActions.getCache(SecurityActions.java:39)
	at org.infinispan.globalstate.impl.VolatileLocalConfigurationStorage.lambda$createCache$0(VolatileLocalConfigurationStorage.java:87)
	at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1700)
	at org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35)
	at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1982)
	at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1486)
	at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1377)
	at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: java.util.concurrent.CompletionException: org.infinispan.persistence.spi.PersistenceException: java.lang.IllegalStateException: Error reading header from 1:552537 | 0
	at org.infinispan.util.concurrent.CompletionStages.join(CompletionStages.java:85)
	at org.infinispan.persistence.sifs.NonBlockingSoftIndexFileStore.buildIndex(NonBlockingSoftIndexFileStore.java:395)
	at org.infinispan.persistence.sifs.NonBlockingSoftIndexFileStore.lambda$start$0(NonBlockingSoftIndexFileStore.java:239)
	at org.infinispan.util.concurrent.BlockingManagerImpl.runBlockingOperation(BlockingManagerImpl.java:109)
	at org.infinispan.util.concurrent.BlockingManagerImpl.runBlockingOperation(BlockingManagerImpl.java:99)
	at org.infinispan.util.concurrent.BlockingManagerImpl.runBlocking(BlockingManagerImpl.java:65)
	at org.infinispan.persistence.sifs.NonBlockingSoftIndexFileStore.start(NonBlockingSoftIndexFileStore.java:215)
	at org.infinispan.configuration.parsing.SFSToSIFSStore.createAndStartStore(SFSToSIFSStore.java:96)
	at org.infinispan.configuration.parsing.SFSToSIFSStore.createAndStartSoftIndexFileStore(SFSToSIFSStore.java:90)
	at org.infinispan.configuration.parsing.SFSToSIFSStore.start(SFSToSIFSStore.java:49)
	at org.infinispan.persistence.manager.PersistenceManagerImpl.lambda$startNewStores$4(PersistenceManagerImpl.java:220)
	at io.reactivex.rxjava3.internal.operators.mixed.FlowableConcatMapSingle$ConcatMapSingleSubscriber.drain(FlowableConcatMapSingle.java:249)
	at io.reactivex.rxjava3.internal.operators.mixed.FlowableConcatMapSingle$ConcatMapSingleSubscriber.onNext(FlowableConcatMapSingle.java:136)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable$IteratorSubscription.slowPath(FlowableFromIterable.java:243)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable$BaseRangeSubscription.request(FlowableFromIterable.java:131)
	at io.reactivex.rxjava3.internal.operators.mixed.FlowableConcatMapSingle$ConcatMapSingleSubscriber.onSubscribe(FlowableConcatMapSingle.java:125)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable.subscribe(FlowableFromIterable.java:69)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable.subscribeActual(FlowableFromIterable.java:47)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750)
	at io.reactivex.rxjava3.internal.operators.mixed.FlowableConcatMapSingle.subscribeActual(FlowableConcatMapSingle.java:61)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableDoOnEach.subscribeActual(FlowableDoOnEach.java:50)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFlatMap.subscribeActual(FlowableFlatMap.java:52)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableIgnoreElementsCompletable.subscribeActual(FlowableIgnoreElementsCompletable.java:34)
	at io.reactivex.rxjava3.core.Completable.subscribe(Completable.java:2850)
	at io.reactivex.rxjava3.internal.operators.completable.CompletablePeek.subscribeActual(CompletablePeek.java:51)
	at io.reactivex.rxjava3.core.Completable.subscribe(Completable.java:2850)
	at io.reactivex.rxjava3.core.Completable.blockingAwait(Completable.java:1460)
	at org.infinispan.persistence.manager.PersistenceManagerImpl.start(PersistenceManagerImpl.java:185)
	at org.infinispan.persistence.manager.CorePackageImpl$3.start(CorePackageImpl.java:85)
	at org.infinispan.persistence.manager.CorePackageImpl$3.start(CorePackageImpl.java:64)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:617)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:608)
	at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:577)
	... 43 more
Caused by: org.infinispan.persistence.spi.PersistenceException: java.lang.IllegalStateException: Error reading header from 1:552537 | 0
	at org.infinispan.persistence.sifs.NonBlockingSoftIndexFileStore$HandleIterator.getNext(NonBlockingSoftIndexFileStore.java:847)
	at org.infinispan.commons.util.AbstractIterator.hasNext(AbstractIterator.java:26)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable.subscribe(FlowableFromIterable.java:53)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable.subscribeActual(FlowableFromIterable.java:47)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15696)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableUsing.subscribeActual(FlowableUsing.java:73)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15696)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFlatMap$MergeSubscriber.onNext(FlowableFlatMap.java:162)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableUsing$UsingSubscriber.onNext(FlowableUsing.java:104)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable$IteratorSubscription.slowPath(FlowableFromIterable.java:243)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable$BaseRangeSubscription.request(FlowableFromIterable.java:131)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableUsing$UsingSubscriber.request(FlowableUsing.java:157)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFlatMap$MergeSubscriber.onSubscribe(FlowableFlatMap.java:116)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableUsing$UsingSubscriber.onSubscribe(FlowableUsing.java:98)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable.subscribe(FlowableFromIterable.java:69)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable.subscribeActual(FlowableFromIterable.java:47)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15696)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableUsing.subscribeActual(FlowableUsing.java:73)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFlatMap.subscribeActual(FlowableFlatMap.java:52)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableDoOnEach.subscribeActual(FlowableDoOnEach.java:50)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15696)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFlatMap$MergeSubscriber.onNext(FlowableFlatMap.java:162)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableUsing$UsingSubscriber.onNext(FlowableUsing.java:104)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable$IteratorSubscription.slowPath(FlowableFromIterable.java:243)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable$BaseRangeSubscription.request(FlowableFromIterable.java:131)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableUsing$UsingSubscriber.request(FlowableUsing.java:157)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFlatMap$MergeSubscriber.onSubscribe(FlowableFlatMap.java:116)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableUsing$UsingSubscriber.onSubscribe(FlowableUsing.java:98)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable.subscribe(FlowableFromIterable.java:69)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable.subscribeActual(FlowableFromIterable.java:47)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15696)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableUsing.subscribeActual(FlowableUsing.java:73)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableFlatMap.subscribeActual(FlowableFlatMap.java:52)
	at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750)
	at io.reactivex.rxjava3.internal.operators.flowable.FlowableIgnoreElementsCompletable.subscribeActual(FlowableIgnoreElementsCompletable.java:34)
	at io.reactivex.rxjava3.core.Completable.subscribe(Completable.java:2850)
	at io.reactivex.rxjava3.core.Completable.subscribeWith(Completable.java:2900)
	at io.reactivex.rxjava3.core.Completable.toCompletionStage(Completable.java:3424)
	at org.infinispan.persistence.sifs.NonBlockingSoftIndexFileStore.buildIndex(NonBlockingSoftIndexFileStore.java:394)
	... 77 more
Caused by: java.lang.IllegalStateException: Error reading header from 1:552537 | 0
	at org.infinispan.persistence.sifs.IndexNode$LeafNode.getHeaderAndKey(IndexNode.java:1035)
	at org.infinispan.persistence.sifs.IndexNode$LeafNode.loadHeaderAndKey(IndexNode.java:1015)
	at org.infinispan.persistence.sifs.IndexNode$ReadOperation$4.apply(IndexNode.java:256)
	at org.infinispan.persistence.sifs.IndexNode$ReadOperation$4.apply(IndexNode.java:253)
	at org.infinispan.persistence.sifs.IndexNode.applyOnLeaf(IndexNode.java:295)
	at org.infinispan.persistence.sifs.Index.getInfo(Index.java:146)
	at org.infinispan.persistence.sifs.NonBlockingSoftIndexFileStore.isSeqIdOld(NonBlockingSoftIndexFileStore.java:415)
	at org.infinispan.persistence.sifs.NonBlockingSoftIndexFileStore.lambda$buildIndex$3(NonBlockingSoftIndexFileStore.java:385)
	at org.infinispan.persistence.sifs.NonBlockingSoftIndexFileStore$HandleIterator.getNext(NonBlockingSoftIndexFileStore.java:839)
	... 123 more

clones

JDG-5032 [Operator] Scaling 3->1->3->0->3 may crash cache

Closed

Details

Description

Attachments

Issue Links

Easy Agile Planning Poker

Activity

People

Dates