Hi,
I am upgradingJBoss cache 3.1.0 with hibernate core 3.3.2.
I am facing a problem related to new feature of invalidating the secondary cache while bulk operations.

Presently the code is invalidating all the the cache regions two times for each bulk operations done using entitymanager.executeupdate method .

First invalidation is made while executing the Query.executeUpdate and Second invalidation is done on org.hibernate.transaction.CacheSynchronization.afterCompletion.

This causing org.jboss.cache.lock.TimeoutException: failure acquiring lock:
due to the fact that the first execution is acquiring the lock(not rleaseing) and secondoperation is trying to acquiring the lock again whith in the same thread of execution.

this seem to be a incomplete implementation of org.hibernate.action.BulkOperationCleanupAction.

following is the stack trace.

at org.jboss.cache.lock.NodeBasedLockManager.acquireLock(NodeBasedLockManager.java:87)
at org.jboss.cache.lock.NodeBasedLockManager.lock(NodeBasedLockManager.java:117)
at org.jboss.cache.interceptors.OptimisticInterceptor.lockAndCreateWorkspaceNode(OptimisticInterceptor.java:134)
at org.jboss.cache.interceptors.OptimisticNodeInterceptor.fetchWorkspaceNode(OptimisticNodeInterceptor.java:583)
at org.jboss.cache.interceptors.OptimisticNodeInterceptor.visitRemoveNodeCommand(OptimisticNodeInterceptor.java:103)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.EvictionInterceptor.visitRemoveNodeCommand(EvictionInterceptor.java:223)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)
at org.jboss.cache.commands.AbstractVisitor.visitRemoveNodeCommand(AbstractVisitor.java:75)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)
at org.jboss.cache.commands.AbstractVisitor.visitRemoveNodeCommand(AbstractVisitor.java:75)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)
at org.jboss.cache.commands.AbstractVisitor.visitRemoveNodeCommand(AbstractVisitor.java:75)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.InvalidationInterceptor.handleWriteMethod(InvalidationInterceptor.java:241)
at org.jboss.cache.interceptors.InvalidationInterceptor.visitRemoveNodeCommand(InvalidationInterceptor.java:125)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)
at org.jboss.cache.commands.AbstractVisitor.visitRemoveNodeCommand(AbstractVisitor.java:75)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.TxInterceptor.attachGtxAndPassUpChain(TxInterceptor.java:301)
at org.jboss.cache.interceptors.OptimisticTxInterceptor.handleDefault(OptimisticTxInterceptor.java:83)
at org.jboss.cache.commands.AbstractVisitor.visitRemoveNodeCommand(AbstractVisitor.java:75)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)
at org.jboss.cache.commands.AbstractVisitor.visitRemoveNodeCommand(AbstractVisitor.java:75)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:178)
at org.jboss.cache.interceptors.InvocationContextInterceptor.visitRemoveNodeCommand(InvocationContextInterceptor.java:88)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.InterceptorChain.invoke(InterceptorChain.java:287)
at org.jboss.cache.invocation.CacheInvocationDelegate.removeNode(CacheInvocationDelegate.java:478)
at org.hibernate.cache.jbc2.util.CacheHelper.removeAll(CacheHelper.java:378)
at org.hibernate.cache.jbc2.access.OptimisticTransactionalAccessDelegate.evictOrRemoveAll(OptimisticTransactionalAccessDelegate.java:172)
at org.hibernate.cache.jbc2.access.OptimisticTransactionalAccessDelegate.evictAll(OptimisticTransactionalAccessDelegate.java:80)
at org.hibernate.cache.jbc2.entity.TransactionalAccess.evictAll(TransactionalAccess.java:102)
at org.hibernate.impl.SessionFactoryImpl.evictEntity(SessionFactoryImpl.java:918)
at org.hibernate.action.BulkOperationCleanupAction.evictEntityRegions(BulkOperationCleanupAction.java:153)
at org.hibernate.action.BulkOperationCleanupAction.afterTransactionCompletion(BulkOperationCleanupAction.java:132)
at org.hibernate.engine.ActionQueue.afterTransactionCompletion(ActionQueue.java:198)
at org.hibernate.impl.SessionImpl.afterTransactionCompletion(SessionImpl.java:452)
at org.hibernate.jdbc.JDBCContext.afterTransactionCompletion(JDBCContext.java:252)
at org.hibernate.transaction.CacheSynchronization.afterCompletion(CacheSynchronization.java:117)
at weblogic.transaction.internal.ServerSCInfo.doAfterCompletion(ServerSCInfo.java:1032)
at weblogic.transaction.internal.ServerSCInfo.callAfterCompletions(ServerSCInfo.java:1011)
at weblogic.transaction.internal.ServerTransactionImpl.callAfterCompletions(ServerTransactionImpl.java:2946)


Is there any workaround for the issue?

browsing through the hibernate jira i have seen HHH-4034 is trying to address the issue indirectly
but after trying the fix another problem

java.lang.UnsupportedOperationException: Illegal attempt to edit read only region
at org.hibernate.cache.jbc2.entity.OptimisticReadOnlyAccess.lockRegion(OptimisticReadOnlyAccess.java:54)
at org.hibernate.action.BulkOperationCleanupAction$EntityCleanup.<init>(BulkOperationCleanupAction.java:223)
at org.hibernate.action.BulkOperationCleanupAction$EntityCleanup.<init>(BulkOperationCleanupAction.java:217)
at org.hibernate.action.BulkOperationCleanupAction.<init>(BulkOperationCleanupAction.java:132)
at org.hibernate.engine.query.NativeSQLQueryPlan.coordinateSharedCacheCleanup(NativeSQLQueryPlan.java:176)
at org.hibernate.engine.query.NativeSQLQueryPlan.performExecuteUpdate(NativeSQLQueryPlan.java:189)
at org.hibernate.impl.SessionImpl.executeNativeUpdate(SessionImpl.java:1191)
at org.hibernate.impl.SQLQueryImpl.executeUpdate(SQLQueryImpl.java:357)
at org.hibernate.ejb.QueryImpl.executeUpdate(QueryImpl.java:51)

is enbody able to guide how i can work around this problem?

http://opensource.atlassian.com/project ... e/HHH-2224 is also similar issue but it specified as fixed. but it looks like it is the same issue

Even I have tried with the new mvcc locking scehme.. its look like the problem is still there.
the exception log is follows

k held by [GlobalTransaction:<192.168.10.49:33016>:13]
at org.hibernate.cache.jbc2.util.CacheHelper.removeAll(CacheHelper.java:380)
at org.hibernate.cache.jbc2.util.CacheHelper.removeAll(CacheHelper.java:360)
at org.hibernate.cache.jbc2.access.TransactionalAccessDelegate.evictOrRemoveAll(TransactionalAccessDelegate.java:146)
at org.hibernate.cache.jbc2.access.TransactionalAccessDelegate.evictAll(TransactionalAccessDelegate.java:142)
at org.hibernate.cache.jbc2.entity.TransactionalAccess.evictAll(TransactionalAccess.java:102)
at org.hibernate.impl.SessionFactoryImpl.evictEntity(SessionFactoryImpl.java:918)
at org.hibernate.action.BulkOperationCleanupAction.evictEntityRegions(BulkOperationCleanupAction.java:153)
at org.hibernate.action.BulkOperationCleanupAction.afterTransactionCompletion(BulkOperationCleanupAction.java:132)
at org.hibernate.engine.ActionQueue.afterTransactionCompletion(ActionQueue.java:198)
at org.hibernate.impl.SessionImpl.afterTransactionCompletion(SessionImpl.java:452)
at org.hibernate.jdbc.JDBCContext.afterTransactionCompletion(JDBCContext.java:252)
at org.hibernate.transaction.CacheSynchronization.afterCompletion(CacheSynchronization.java:117)
at weblogic.transaction.internal.ServerSCInfo.doAfterCompletion(ServerSCInfo.java:1032)
at weblogic.transaction.internal.ServerSCInfo.callAfterCompletions(ServerSCInfo.java:1011)
at weblogic.transaction.internal.ServerTransactionImpl.callAfterCompletions(ServerTransactionImpl.java:2946)
at weblogic.transaction.internal.ServerTransactionImpl.afterCommittedStateHousekeeping(ServerTransactionImpl.java:2846)
at weblogic.transaction.internal.ServerTransactionImpl.setCommitted(ServerTransactionImpl.java:2891)
at weblogic.transaction.internal.ServerTransactionImpl.globalRetryCommit(ServerTransactionImpl.java:2681)
at weblogic.transaction.internal.ServerTransactionImpl.globalCommit(ServerTransactionImpl.java:2601)
at weblogic.transaction.internal.ServerTransactionImpl.internalCommit(ServerTransactionImpl.java:279)
at weblogic.transaction.internal.ServerTransactionImpl.commit(ServerTransactionImpl.java:227)
at weblogic.ejb.container.internal.BaseLocalObject.postInvoke1(BaseLocalObject.java:588)
at weblogic.ejb.container.internal.BaseLocalObject.postInvokeTxRetry(BaseLocalObject.java:421)
at weblogic.ejb.container.internal.StatefulLocalObject.postInvokeTxRetry(StatefulLocalObject.java:86)
at com.idsgrp.profinia.server.loan.service.LoanScheduleSetupSessionEJB_xe9xy8_LoanScheduleSetupSessionImpl.activateAndSaveLoanSchedule(LoanScheduleSetupSessionEJB_xe9xy8_LoanScheduleSetupSessionImpl.java:7666)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at com.caucho.hessian.server.HessianSkeleton.invoke(HessianSkeleton.java:157)
at com.idsgrp.profinia.web.HessianServlet.service(HessianServlet.java:140)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:226)
at weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:124)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:283)
at weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:26)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:42)
at com.idsgrp.profinia.server.system.TLRegistryCleanUpFilter.doFilter(TLRegistryCleanUpFilter.java:27)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:42)
at com.idsgrp.profinia.server.web.UOWUnregisterFilter.doFilter(UOWUnregisterFilter.java:22)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:42)
at com.idsgrp.profinia.serverutil.web.SessionFilter.doFilter(SessionFilter.java:41)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:42)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3368)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
at weblogic.security.service.SecurityManager.runAs(Unknown Source)
at weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2117)
at weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2023)
at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1359)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:200)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:172)
Caused by: org.jboss.cache.lock.TimeoutException: Unable to acquire lock on Fqn [/com/idsgrp/profinia/server/tax/twsut/TWSUTState/ENTITY] after [15000] milliseconds for requestor [Thread[[ACTIVE] ExecuteThread: '9' for queue: 'weblogic.kernel.Default (self-tuning)',5,Pooled Threads]]! Lock held by [GlobalTransaction:<192.168.10.49:33016>:13]
at org.jboss.cache.mvcc.MVCCNodeHelper.acquireLock(MVCCNodeHelper.java:157)
at org.jboss.cache.mvcc.MVCCNodeHelper.wrapNodesRecursivelyForRemoval(MVCCNodeHelper.java:326)
at org.jboss.cache.interceptors.MVCCLockingInterceptor.handleRemoveNodeCommand(MVCCLockingInterceptor.java:116)
at org.jboss.cache.interceptors.base.PrePostProcessingCommandInterceptor.visitRemoveNodeCommand(PrePostProcessingCommandInterceptor.java:126)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.InvalidationInterceptor.handleWriteMethod(InvalidationInterceptor.java:241)
at org.jboss.cache.interceptors.InvalidationInterceptor.visitRemoveNodeCommand(InvalidationInterceptor.java:125)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)
at org.jboss.cache.commands.AbstractVisitor.visitRemoveNodeCommand(AbstractVisitor.java:75)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.TxInterceptor.attachGtxAndPassUpChain(TxInterceptor.java:301)
at org.jboss.cache.interceptors.TxInterceptor.handleDefault(TxInterceptor.java:283)
at org.jboss.cache.commands.AbstractVisitor.visitRemoveNodeCommand(AbstractVisitor.java:75)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)
at org.jboss.cache.commands.AbstractVisitor.visitRemoveNodeCommand(AbstractVisitor.java:75)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
at org.jboss.cache.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:178)
at org.jboss.cache.interceptors.InvocationContextInterceptor.visitRemoveNodeCommand(InvocationContextInterceptor.java:88)
at org.jboss.cache.commands.write.RemoveNodeCommand.acceptVisitor(RemoveNodeCommand.java:125)
at org.jboss.cache.interceptors.InterceptorChain.invoke(InterceptorChain.java:287)
at org.jboss.cache.invocation.CacheInvocationDelegate.removeNode(CacheInvocationDelegate.java:478)
at org.hibernate.cache.jbc2.util.CacheHelper.removeAll(CacheHelper.java:378)
... 49 more