[GSS](7.3.z) WFLY-14861 - Session objects left in memory after non-coordinator member left a cluster

1. startup 1 node and generate load of 100 sessions
2. scaleup 2nd node and generate load of 100 sessions
3. scaleup 3rd node and generate load of 100 sessions
4. scaleup 4th node and generate load of 100 sessions
5. scaledown to 3 nodes and generate load of 100 sessions
6. scaledown to 2 nodes and generate load of 100 sessions

Sat Jun 12 00:02:30 JST 2021
PID: 942
# before GC --------------------------
 num #instances #bytes class name (module)
 602: 55 1320 org.wildfly.clustering.web.infinispan.session.fine.SessionAttributeKey
--------------------------------------
# trigger GC to clean up for testing purpose 
942:
Command executed successfully
# after GC ---------------------------
 num #instances #bytes class name (module)
 584: 55 1320 org.wildfly.clustering.web.infinispan.session.fine.SessionAttributeKey
--------------------------------------

ERROR *** JBossAS wrapper process (1) received TERM signal ***
23:44:45,586 INFO [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-6) ISPN000080: Disconnecting JGroups channel ee
23:44:45,588 INFO [org.jboss.as.mail.extension] (MSC service thread 1-1) WFLYMAIL0002: Unbound mail session [java:jboss/mail/Default]
23:44:45,589 INFO [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-2) ISPN000080: Disconnecting JGroups channel ee
23:44:45,590 INFO [org.wildfly.extension.undertow] (ServerService Thread Pool -- 87) WFLYUT0022: Unregistered web context: '/globalmenu-pl-pc' from server 'default-server'
23:44:45,598 INFO [org.wildfly.extension.messaging-activemq] (ServerService Thread Pool -- 7) WFLYMSGAMQ0006: Unbound messaging object to jndi name java:jboss/exported/jms/RemoteConnectionFactory
23:44:45,634 DEBUG [org.jboss.as.clustering.infinispan] (MSC service thread 1-6) hibernate cache container stopped
23:44:45,635 DEBUG [org.jboss.as.clustering.infinispan] (MSC service thread 1-2) server cache container stopped
23:44:45,713 INFO [org.wildfly.extension.undertow] (MSC service thread 1-3) WFLYUT0019: Host default-host stopping
23:44:45,730 INFO [org.jboss.as.connector.deployment] (MSC service thread 1-6) WFLYJCA0119: Unbinding connection factory named java:/JmsXA to alias java:jboss/DefaultJMSConnectionFactory
23:44:45,730 INFO [org.jboss.as.connector.deployment] (MSC service thread 1-6) WFLYJCA0011: Unbound JCA ConnectionFactory [java:/JmsXA]
23:44:45,800 INFO [org.jboss.modcluster] (ServerService Thread Pool -- 90) MODCLUSTER000002: Initiating mod_cluster shutdown
23:44:45,822 INFO [org.jboss.as.clustering.infinispan] (ServerService Thread Pool -- 98) WFLYCLINF0003: Stopped client-mappings cache from ejb container
23:44:45,824 WARN [org.wildfly.clustering.web.infinispan] (SessionExpirationScheduler - 1) WFLYCLWEBINF0004: Failed to expire session ZsIW17pCm80VKA3PEZIu6f8_GGsomCeHATYtvjMfp5b2afw_: org.infinispan.IllegalLifecycleStateException: ISPN000324: Cache 'globalmenu-pl-pc.war' is in 'STOPPING' state and this is an invocation not belonging to an on-going transaction, so it does not accept new invocations. Either restart it or recreate the cache container.
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.interceptors.impl.InvocationContextInterceptor.ignoreCommand(InvocationContextInterceptor.java:101)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:87)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:57)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.interceptors.DDAsyncInterceptor.visitRemoveCommand(DDAsyncInterceptor.java:65)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.commands.write.RemoveCommand.acceptVisitor(RemoveCommand.java:64)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.interceptors.DDAsyncInterceptor.visitCommand(DDAsyncInterceptor.java:50)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:248)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.cache.impl.CacheImpl.executeCommandAndCommitIfNeeded(CacheImpl.java:1915)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.cache.impl.CacheImpl.remove(CacheImpl.java:680)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.cache.impl.DecoratedCache.remove(DecoratedCache.java:559)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.cache.impl.AbstractDelegatingCache.remove(AbstractDelegatingCache.java:453)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.cache.impl.EncoderCache.remove(EncoderCache.java:684)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.cache.impl.AbstractDelegatingCache.remove(AbstractDelegatingCache.java:453)
 at org.wildfly.clustering.web.infinispan@7.3.5.GA-redhat-00001//org.wildfly.clustering.web.infinispan.session.fine.FineSessionAttributesFactory.delete(FineSessionAttributesFactory.java:139)
 at org.wildfly.clustering.web.infinispan@7.3.5.GA-redhat-00001//org.wildfly.clustering.web.infinispan.session.fine.FineSessionAttributesFactory.remove(FineSessionAttributesFactory.java:127)
 at org.wildfly.clustering.web.infinispan@7.3.5.GA-redhat-00001//org.wildfly.clustering.web.infinispan.session.fine.FineSessionAttributesFactory.remove(FineSessionAttributesFactory.java:66)
 at org.wildfly.clustering.web.cache@7.3.5.GA-redhat-00001//org.wildfly.clustering.web.cache.session.CompositeSessionFactory.remove(CompositeSessionFactory.java:89)
 at org.wildfly.clustering.web.cache@7.3.5.GA-redhat-00001//org.wildfly.clustering.web.cache.session.CompositeSessionFactory.remove(CompositeSessionFactory.java:39)
 at org.wildfly.clustering.web.infinispan@7.3.5.GA-redhat-00001//org.wildfly.clustering.web.infinispan.session.ExpiredSessionRemover.remove(ExpiredSessionRemover.java:65)
 at org.wildfly.clustering.web.infinispan@7.3.5.GA-redhat-00001//org.wildfly.clustering.web.infinispan.session.ExpiredSessionRemover.remove(ExpiredSessionRemover.java:41)
 at org.wildfly.clustering.web.infinispan@7.3.5.GA-redhat-00001//org.wildfly.clustering.web.infinispan.session.SessionExpirationScheduler$ExpirationTask.run(SessionExpirationScheduler.java:165)
 at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
 at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
 at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
 at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
 at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
 at java.base/java.lang.Thread.run(Thread.java:834)
 at org.jboss.threads@2.3.3.Final-redhat-00001//org.jboss.threads.JBossThread.run(JBossThread.java:485)
 Suppressed: org.infinispan.util.logging.TraceException
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.interceptors.impl.SimpleAsyncInvocationStage.get(SimpleAsyncInvocationStage.java:41)
 at org.infinispan@9.4.19.Final-redhat-00001//org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:250)
 ... 21 more