Managed to get a client VM to deadlock a few times after restart. Instantiators were implemented in static initializers as documented in API.
Should instantiators be explicitly and serially invoked before allowing access to the cache?
Found one Java-level deadlock:
==============================
"http-8081-219" nid=323 state=BLOCKED
- waiting on <0x6aed299a> (a com.gemstone.gemfire.cache.client.internal.ConnectionImpl)
- locked <0x6aed299a> (a com.gemstone.gemfire.cache.client.internal.ConnectionImpl)
owned by http-8081-3 id=31
at com.gemstone.gemfire.cache.client.internal.ConnectionImpl.execute(ConnectionImpl.java:220)
at com.gemstone.gemfire.cache.client.internal.pooling.PooledConnection.execute(PooledConnection.java:321)
at com.gemstone.gemfire.cache.client.internal.OpExecutorImpl.executeWithPossibleReAuthentication(OpExecutorImpl.java:646)
at com.gemstone.gemfire.cache.client.internal.OpExecutorImpl.execute(OpExecutorImpl.java:108)
at com.gemstone.gemfire.cache.client.internal.PoolImpl.execute(PoolImpl.java:624)
at com.gemstone.gemfire.cache.client.internal.RegisterInstantiatorsOp.execute(RegisterInstantiatorsOp.java:40)
at com.gemstone.gemfire.internal.cache.PoolManagerImpl.allPoolsRegisterInstantiator(PoolManagerImpl.java:219)
at com.gemstone.gemfire.internal.InternalInstantiator.sendRegistrationMessageToServers(InternalInstantiator.java:204)
at com.gemstone.gemfire.internal.InternalInstantiator._register(InternalInstantiator.java:159)
at com.gemstone.gemfire.internal.InternalInstantiator.register(InternalInstantiator.java:87)
- locked java.lang.Class@11e2a4dd
at com.gemstone.gemfire.Instantiator.register(Instantiator.java:168)
- locked java.lang.Class@2f2563c6
at com.me.MyClass.<clinit>(MyClass.java:23)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:247)
at com.gemstone.gemfire.internal.InternalInstantiator.getInstantiator(InternalInstantiator.java:502)
at com.gemstone.gemfire.internal.InternalDataSerializer.readUserDataSerializable(InternalDataSerializer.java:2246)
at com.gemstone.gemfire.internal.InternalDataSerializer.basicReadObject(InternalDataSerializer.java:2186)
at com.gemstone.gemfire.DataSerializer.readObject(DataSerializer.java:3095)
at com.gemstone.gemfire.internal.util.BlobHelper.deserializeBlob(BlobHelper.java:69)
at com.gemstone.gemfire.internal.cache.tier.sockets.CacheServerHelper.deserialize(CacheServerHelper.java:53)
at com.gemstone.gemfire.internal.cache.tier.sockets.Part.getObject(Part.java:178)
at com.gemstone.gemfire.internal.cache.tier.sockets.Part.getObject(Part.java:182)
at com.gemstone.gemfire.cache.client.internal.AbstractOp.processObjResponse(AbstractOp.java:274)
at com.gemstone.gemfire.cache.client.internal.GetOp$GetOpImpl.processResponse(GetOp.java:160)
at com.gemstone.gemfire.cache.client.internal.AbstractOp.attemptReadResponse(AbstractOp.java:163)
at com.gemstone.gemfire.cache.client.internal.AbstractOp.attempt(AbstractOp.java:363)
at com.gemstone.gemfire.cache.client.internal.ConnectionImpl.execute(ConnectionImpl.java:229)
- locked com.gemstone.gemfire.cache.client.internal.ConnectionImpl@6d4767c2
at com.gemstone.gemfire.cache.client.internal.pooling.PooledConnection.execute(PooledConnection.java:321)
at com.gemstone.gemfire.cache.client.internal.OpExecutorImpl.executeWithPossibleReAuthentication(OpExecutorImpl.java:646)
at com.gemstone.gemfire.cache.client.internal.OpExecutorImpl.execute(OpExecutorImpl.java:108)
at com.gemstone.gemfire.cache.client.internal.OpExecutorImpl.execute(OpExecutorImpl.java:84)
at com.gemstone.gemfire.cache.client.internal.PoolImpl.execute(PoolImpl.java:610)
at com.gemstone.gemfire.cache.client.internal.GetOp.execute(GetOp.java:72)
at com.gemstone.gemfire.cache.client.internal.ServerRegionProxy.get(ServerRegionProxy.java:120)
at com.gemstone.gemfire.internal.cache.LocalRegion.findObjectInSystem(LocalRegion.java:2291)
at com.gemstone.gemfire.internal.cache.LocalRegion.nonTxnFindObject(LocalRegion.java:1318)
at com.gemstone.gemfire.internal.cache.LocalRegionDataView.findObject(LocalRegionDataView.java:99)
at com.gemstone.gemfire.internal.cache.LocalRegion.get(LocalRegion.java:1226)
at com.gemstone.gemfire.internal.cache.AbstractRegion.get(AbstractRegion.java:234)
at sun.reflect.GeneratedMethodAccessor104.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.springframework.data.gemfire.GemfireTemplate$CloseSuppressingInvocationHandler.invoke(GemfireTemplate.java:410)
at $Proxy8.get(Unknown Source)
...
"http-8081-3" nid=31 state=BLOCKED
- waiting on <0x11e2a4dd> (a java.lang.Class)
- locked <0x11e2a4dd> (a java.lang.Class)
owned by http-8081-219 id=323
at com.gemstone.gemfire.internal.InternalInstantiator.getInstantiator(InternalInstantiator.java:506)
at com.gemstone.gemfire.internal.InternalDataSerializer.readUserDataSerializable(InternalDataSerializer.java:2246)
at com.gemstone.gemfire.internal.InternalDataSerializer.basicReadObject(InternalDataSerializer.java:2186)
at com.gemstone.gemfire.DataSerializer.readObject(DataSerializer.java:3095)
at com.gemstone.gemfire.internal.util.BlobHelper.deserializeBlob(BlobHelper.java:69)
at com.gemstone.gemfire.internal.cache.tier.sockets.CacheServerHelper.deserialize(CacheServerHelper.java:53)
at com.gemstone.gemfire.internal.cache.tier.sockets.Part.getObject(Part.java:178)
at com.gemstone.gemfire.internal.cache.tier.sockets.Part.getObject(Part.java:182)
at com.gemstone.gemfire.cache.client.internal.AbstractOp.processObjResponse(AbstractOp.java:274)
at com.gemstone.gemfire.cache.client.internal.GetOp$GetOpImpl.processResponse(GetOp.java:160)
at com.gemstone.gemfire.cache.client.internal.AbstractOp.attemptReadResponse(AbstractOp.java:163)
at com.gemstone.gemfire.cache.client.internal.AbstractOp.attempt(AbstractOp.java:363)
at com.gemstone.gemfire.cache.client.internal.ConnectionImpl.execute(ConnectionImpl.java:229)
at com.gemstone.gemfire.cache.client.internal.pooling.PooledConnection.execute(PooledConnection.java:321)
at com.gemstone.gemfire.cache.client.internal.OpExecutorImpl.executeWithPossibleReAuthentication(OpExecutorImpl.java:646)
at com.gemstone.gemfire.cache.client.internal.OpExecutorImpl.execute(OpExecutorImpl.java:108)
at com.gemstone.gemfire.cache.client.internal.OpExecutorImpl.execute(OpExecutorImpl.java:84)
at com.gemstone.gemfire.cache.client.internal.PoolImpl.execute(PoolImpl.java:610)
at com.gemstone.gemfire.cache.client.internal.GetOp.execute(GetOp.java:72)
at com.gemstone.gemfire.cache.client.internal.ServerRegionProxy.get(ServerRegionProxy.java:120)
at com.gemstone.gemfire.internal.cache.LocalRegion.findObjectInSystem(LocalRegion.java:2291)
at com.gemstone.gemfire.internal.cache.LocalRegion.nonTxnFindObject(LocalRegion.java:1318)
at com.gemstone.gemfire.internal.cache.LocalRegionDataView.findObject(LocalRegionDataView.java:99)
at com.gemstone.gemfire.internal.cache.LocalRegion.get(LocalRegion.java:1226)
at com.gemstone.gemfire.internal.cache.AbstractRegion.get(AbstractRegion.java:234)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.springframework.data.gemfire.GemfireTemplate$CloseSuppressingInvocationHandler.invoke(GemfireTemplate.java:410)
at $Proxy8.get(Unknown Source)
...