BEA-000394 DEADLOCK DETECTED

Some days ago a managed server went boom crash bang during a restart,
and there was an elegant message warning us of a deadlock.

It's a lovely feature, but I still need to understand which service is monitoring the thread health.


<Sep 14, 2011 9:29:27 AM BST> <Critical> <WebLogicServer> <BEA-000394> <



DEADLOCK DETECTED:

==================



[deadlocked thread] [ACTIVE] ExecuteThread: '199' for queue: 'weblogic.kernel.Default (self-tuning)':

----------------------------------------------------------------------------------------------------

Thread '[ACTIVE] ExecuteThread: '199' for queue: 'weblogic.kernel.Default (self-tuning)'' is waiting to acquire lock 'java.util.concurrent.ConcurrentHashMap@d084be' that is held by thread '[ACTIVE] ExecuteThread: '15' for queue: 'weblogic.kernel.Default (self-tuning)''



Stack trace:

------------

weblogic.jndi.internal.BasicNamingNode.unbindHere(BasicNamingNode.java:587)

weblogic.jndi.internal.ServerNamingNode.unbindHere(ServerNamingNode.java:328)

weblogic.jndi.internal.BasicNamingNode.unbind(BasicNamingNode.java:561)

weblogic.jndi.internal.BasicNamingNode.unbind(BasicNamingNode.java:573)

weblogic.jndi.internal.BasicNamingNode.unbind(BasicNamingNode.java:573)

weblogic.jndi.internal.BasicNamingNode.unbind(BasicNamingNode.java:573)

weblogic.jndi.internal.BasicNamingNode.unbind(BasicNamingNode.java:573)

weblogic.jndi.internal.WLEventContextImpl.unbind(WLEventContextImpl.java:173)

weblogic.messaging.common.PrivilegedActionUtilities$4.run(PrivilegedActionUtilities.java:102)

weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:363)

weblogic.security.service.SecurityManager.runAs(Unknown Source)

weblogic.messaging.common.PrivilegedActionUtilities.unbindAsSU(PrivilegedActionUtilities.java:99)

weblogic.jms.common.SingularAggregatableManager.singularUnbind(SingularAggregatableManager.java:396)

weblogic.jms.frontend.FEConnection.removeClientID(FEConnection.java:387)

weblogic.jms.frontend.FEConnection.close(FEConnection.java:551)

weblogic.jms.frontend.FEConnection.normalClose(FEConnection.java:404)

weblogic.jms.frontend.FEConnection.invoke(FEConnection.java:1623)

weblogic.messaging.dispatcher.Request.wrappedFiniteStateMachine(Request.java:961)

weblogic.messaging.dispatcher.DispatcherServerRef.invoke(DispatcherServerRef.java:276)

weblogic.messaging.dispatcher.DispatcherServerRef.handleRequest(DispatcherServerRef.java:141)

weblogic.messaging.dispatcher.DispatcherServerRef.access$000(DispatcherServerRef.java:34)

weblogic.messaging.dispatcher.DispatcherServerRef$2.run(DispatcherServerRef.java:111)

weblogic.work.ExecuteThread.execute(ExecuteThread.java:201)

weblogic.work.ExecuteThread.run(ExecuteThread.java:173)



[deadlocked thread] [ACTIVE] ExecuteThread: '15' for queue: 'weblogic.kernel.Default (self-tuning)':

---------------------------------------------------------------------------------------------------

Thread '[ACTIVE] ExecuteThread: '15' for queue: 'weblogic.kernel.Default (self-tuning)'' is waiting to acquire lock 'weblogic.cluster.ServiceAdvertiser@112bb58' that is held by thread '[ACTIVE] ExecuteThread: '9' for queue: 'weblogic.kernel.Default (self-tuning)''



Stack trace:

------------

weblogic.cluster.ServiceAdvertiser.announceReplacement(ServiceAdvertiser.java:132)

weblogic.cluster.ServiceAdvertiser.retractService(ServiceAdvertiser.java:95)

weblogic.jndi.internal.ServerNamingNode.advertiseUnbinding(ServerNamingNode.java:347)

weblogic.jndi.internal.ServerNamingNode.unbindHere(ServerNamingNode.java:330)

weblogic.jndi.internal.BasicNamingNode.unbind(BasicNamingNode.java:561)

weblogic.jndi.internal.BasicNamingNode.unbind(BasicNamingNode.java:573)

weblogic.jndi.internal.BasicNamingNode.unbind(BasicNamingNode.java:573)

weblogic.jndi.internal.BasicNamingNode.unbind(BasicNamingNode.java:573)

weblogic.jndi.internal.BasicNamingNode.unbind(BasicNamingNode.java:573)

weblogic.jndi.internal.WLEventContextImpl.unbind(WLEventContextImpl.java:173)

weblogic.messaging.common.PrivilegedActionUtilities$4.run(PrivilegedActionUtilities.java:102)

weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:363)

weblogic.security.service.SecurityManager.runAs(Unknown Source)

weblogic.messaging.common.PrivilegedActionUtilities.unbindAsSU(PrivilegedActionUtilities.java:99)

weblogic.jms.common.SingularAggregatableManager.singularUnbind(SingularAggregatableManager.java:396)

weblogic.jms.common.SingularAggregatableManager.aggregatableDidBind(SingularAggregatableManager.java:479)

weblogic.jms.common.SingularAggregatable.onBind(SingularAggregatable.java:167)

weblogic.jndi.internal.BasicNamingNode.bindHere(BasicNamingNode.java:371)

weblogic.jndi.internal.ServerNamingNode.bindHere(ServerNamingNode.java:140)

weblogic.jndi.internal.BasicNamingNode.bind(BasicNamingNode.java:317)

weblogic.jndi.internal.BasicNamingNode.bind(BasicNamingNode.java:324)

weblogic.jndi.internal.BasicNamingNode.bind(BasicNamingNode.java:324)

weblogic.jndi.internal.BasicNamingNode.bind(BasicNamingNode.java:324)

weblogic.jndi.internal.BasicNamingNode.bind(BasicNamingNode.java:324)

weblogic.jndi.internal.WLEventContextImpl.bind(WLEventContextImpl.java:277)

weblogic.cluster.BasicServiceOffer.install(BasicServiceOffer.java:101)

weblogic.cluster.NodeInfo.install(NodeInfo.java:62)

weblogic.cluster.TreeManager.install(TreeManager.java:59)

weblogic.cluster.MemberServices.processOffer(MemberServices.java:64)

weblogic.cluster.RemoteMemberInfo.processAnnouncement(RemoteMemberInfo.java:297)

weblogic.cluster.AnnouncementManager$1.run(AnnouncementManager.java:195)

weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)

weblogic.security.service.SecurityManager.runAs(Unknown Source)

weblogic.cluster.AnnouncementManager.receiveAnnouncement(AnnouncementManager.java:186)

weblogic.cluster.AnnouncementMessage.execute(AnnouncementMessage.java:26)

weblogic.cluster.MulticastReceiver$1.run(MulticastReceiver.java:112)

weblogic.work.ExecuteThread.execute(ExecuteThread.java:201)

weblogic.work.ExecuteThread.run(ExecuteThread.java:173)



[deadlocked thread] [ACTIVE] ExecuteThread: '9' for queue: 'weblogic.kernel.Default (self-tuning)':

--------------------------------------------------------------------------------------------------

Thread '[ACTIVE] ExecuteThread: '9' for queue: 'weblogic.kernel.Default (self-tuning)'' is waiting to acquire lock 'weblogic.cluster.NodeInfo@719870' that is held by thread '[ACTIVE] ExecuteThread: '15' for queue: 'weblogic.kernel.Default (self-tuning)''



Stack trace:

------------

weblogic.cluster.NodeInfo.install(NodeInfo.java:61)

weblogic.cluster.TreeManager.install(TreeManager.java:59)

weblogic.cluster.MemberServices.processOffer(MemberServices.java:64)

weblogic.cluster.AnnouncementManager.announce(AnnouncementManager.java:126)

weblogic.cluster.ServiceAdvertiser.announceOffer(ServiceAdvertiser.java:111)

weblogic.cluster.ServiceAdvertiser.offerService(ServiceAdvertiser.java:71)

weblogic.cluster.ServiceAdvertiser.offerService(ServiceAdvertiser.java:46)

weblogic.jndi.internal.ServerNamingNode.advertiseBinding(ServerNamingNode.java:156)

weblogic.jndi.internal.ServerNamingNode.bindHere(ServerNamingNode.java:142)

weblogic.jndi.internal.BasicNamingNode.bind(BasicNamingNode.java:317)

weblogic.jndi.internal.BasicNamingNode.bind(BasicNamingNode.java:324)

weblogic.jndi.internal.BasicNamingNode.bind(BasicNamingNode.java:324)

weblogic.jndi.internal.BasicNamingNode.bind(BasicNamingNode.java:324)

weblogic.jndi.internal.BasicNamingNode.bind(BasicNamingNode.java:324)

weblogic.jndi.internal.WLEventContextImpl.bind(WLEventContextImpl.java:277)

weblogic.messaging.common.PrivilegedActionUtilities$3.run(PrivilegedActionUtilities.java:78)

weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:363)

weblogic.security.service.SecurityManager.runAs(Unknown Source)

weblogic.messaging.common.PrivilegedActionUtilities.bindAsSU(PrivilegedActionUtilities.java:75)

weblogic.jms.common.SingularAggregatableManager.singularBindFinish(SingularAggregatableManager.java:325)

weblogic.jms.frontend.FEConnection.setConnectionClientId(FEConnection.java:1165)

weblogic.jms.frontend.FEConnection.invoke(FEConnection.java:1613)

weblogic.messaging.dispatcher.Request.wrappedFiniteStateMachine(Request.java:961)

weblogic.messaging.dispatcher.DispatcherServerRef.invoke(DispatcherServerRef.java:276)

weblogic.messaging.dispatcher.DispatcherServerRef.handleRequest(DispatcherServerRef.java:141)

weblogic.messaging.dispatcher.DispatcherServerRef.access$000(DispatcherServerRef.java:34)

weblogic.messaging.dispatcher.DispatcherServerRef$2.run(DispatcherServerRef.java:111)

weblogic.work.ExecuteThread.execute(ExecuteThread.java:201)

weblogic.work.ExecuteThread.run(ExecuteThread.java:173)



>