From 41252b3211d644eefb3a67a33b5dd5df4f3de426 Mon Sep 17 00:00:00 2001 From: Suranjan Kumar Date: Wed, 31 Jan 2018 21:18:57 +0530 Subject: [PATCH 01/10] Take pendingTxRegionStates lock in TXBatchMessage This is to avoid a race where GII first takes pendingTXRegionState lock and then tries to apply the operation which in turn tries to create the TXRegionState. In case of BatchMessage, we first create the TXRegionState and then try to take lock on pendingTXRegionState. This causes cyclic dependency and deadlock --- .../gemstone/gemfire/internal/cache/ImageState.java | 4 ++++ .../gemfire/internal/cache/TXBatchMessage.java | 7 +++++++ .../gemfire/internal/cache/TXRegionState.java | 10 ++++++++-- .../gemfire/internal/cache/UnsharedImageState.java | 11 +++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java index 3c7112037..ccf34a46e 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java @@ -187,5 +187,9 @@ public interface VersionTagEntry { public VersionSource getMemberID(); public long getRegionVersion(); } + + public boolean isPendingTXRegionStatesWriteLocked(); + + public Thread getPendingTXRegionStatesLockOwner(); } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java index 146504496..dd111432f 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java @@ -157,6 +157,13 @@ final void apply(final TXStateProxy tx) { region = baseRegion = null; } final int numOps = this.pendingOps.size(); + // take pendingTXRegionStates lock first so that + // GII thread doesn't block on TXRegionState. + for (LocalRegion r : pendingOpsRegions) { + if (!r.isInitialized()) + r.getImageState().lockPendingTXRegionStates(true, false); + } + for (int index = 0; index < numOps; index++) { entry = this.pendingOps.get(index); if (pendingOpsRegion == null) { diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java index 46642f150..abb57f5e0 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java @@ -153,7 +153,14 @@ public TXRegionState(final LocalRegion r, final TXState tx) { this.expiryReadLock = r.getTxEntryExpirationReadLock(); this.isValid = true; - if (!r.isInitialized() && r.getImageState().lockPendingTXRegionStates(true, false)) { + + // check if lock already taken by the same thread + // in that case don't take lock + ImageState imgState = r.getImageState(); + if (!r.isInitialized() + && ((imgState.isPendingTXRegionStatesWriteLocked() + && imgState.getPendingTXRegionStatesLockOwner().getId() == Thread.currentThread().getId()) + || imgState.lockPendingTXRegionStates(true, false))) { try { if (!r.getImageState().addPendingTXRegionState(this)) { this.pendingTXOps = null; @@ -165,7 +172,6 @@ public TXRegionState(final LocalRegion r, final TXState tx) { } finally { r.getImageState().unlockPendingTXRegionStates(true); } - } else { this.pendingTXOps = null; this.pendingTXLockFlags = null; diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java index 6222906fa..208661606 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java @@ -37,6 +37,7 @@ import java.util.Iterator; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Used on distributed replicated regions to track GII and various state. @@ -133,6 +134,16 @@ public void setRequestedUnappliedDelta(boolean flag) { requestedDelta = flag; } + @Override + public boolean isPendingTXRegionStatesWriteLocked() { + return this.pendingTXRegionStatesLock.isWriteLocked(); + } + + @Override + public Thread getPendingTXRegionStatesLockOwner() { + return pendingTXRegionStatesLockOwner; + } + @Override public boolean requestedUnappliedDelta() { return requestedDelta; From 2972e0bda6cdbe007eae22fa4ac2d081fcb0af60 Mon Sep 17 00:00:00 2001 From: Suranjan Kumar Date: Thu, 1 Feb 2018 09:43:36 +0530 Subject: [PATCH 02/10] Unlock the pendingTxRegionStates --- .../internal/cache/TXBatchMessage.java | 47 ++++++++++++------- .../gemfire/internal/cache/TXRegionState.java | 8 ++-- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java index dd111432f..0ea5d7de9 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java @@ -159,29 +159,40 @@ final void apply(final TXStateProxy tx) { final int numOps = this.pendingOps.size(); // take pendingTXRegionStates lock first so that // GII thread doesn't block on TXRegionState. + boolean[] locked = new boolean[pendingOpsRegion.size()]; + int lockedIndex = 0; for (LocalRegion r : pendingOpsRegions) { - if (!r.isInitialized()) - r.getImageState().lockPendingTXRegionStates(true, false); + if (!r.isInitialized()) { + locked[lockedIndex] = r.getImageState().lockPendingTXRegionStates(true, false); + lockedIndex++; + } } - - for (int index = 0; index < numOps; index++) { - entry = this.pendingOps.get(index); - if (pendingOpsRegion == null) { - region = this.pendingOpsRegions.get(index); - if (region.isUsedForPartitionedRegionBucket()) { - baseRegion = region.getPartitionedRegion(); + try { + for (int index = 0; index < numOps; index++) { + entry = this.pendingOps.get(index); + if (pendingOpsRegion == null) { + region = this.pendingOpsRegions.get(index); + if (region.isUsedForPartitionedRegionBucket()) { + baseRegion = region.getPartitionedRegion(); + } else { + baseRegion = region; + } } - else { - baseRegion = region; + if (txState.isCoordinator()) { + region.waitForData(); + } + txrs = txState.writeRegion(region); + if (txrs != null) { + txState.applyPendingOperation(entry, lockFlags, txrs, region, + baseRegion, eventTemplate, true, Boolean.TRUE, this); } } - if (txState.isCoordinator()) { - region.waitForData(); - } - txrs = txState.writeRegion(region); - if (txrs != null) { - txState.applyPendingOperation(entry, lockFlags, txrs, region, - baseRegion, eventTemplate, true, Boolean.TRUE, this); + } finally { + int index = 0; + for (LocalRegion r : pendingOpsRegions) { + if (locked[index]) + r.getImageState().unlockPendingTXRegionStates(true); + index++; } } } finally { diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java index abb57f5e0..08657f597 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java @@ -157,9 +157,10 @@ public TXRegionState(final LocalRegion r, final TXState tx) { // check if lock already taken by the same thread // in that case don't take lock ImageState imgState = r.getImageState(); + boolean alreadyLocked = (imgState.isPendingTXRegionStatesWriteLocked() + && imgState.getPendingTXRegionStatesLockOwner().getId() == Thread.currentThread().getId()); if (!r.isInitialized() - && ((imgState.isPendingTXRegionStatesWriteLocked() - && imgState.getPendingTXRegionStatesLockOwner().getId() == Thread.currentThread().getId()) + && (alreadyLocked || imgState.lockPendingTXRegionStates(true, false))) { try { if (!r.getImageState().addPendingTXRegionState(this)) { @@ -170,7 +171,8 @@ public TXRegionState(final LocalRegion r, final TXState tx) { this.pendingTXLockFlags = new TIntArrayList(); } } finally { - r.getImageState().unlockPendingTXRegionStates(true); + if (!alreadyLocked) + r.getImageState().unlockPendingTXRegionStates(true); } } else { this.pendingTXOps = null; From 8f7d0b1b9c9025fb043ae09106d0e80653256107 Mon Sep 17 00:00:00 2001 From: Suranjan Kumar Date: Thu, 1 Feb 2018 12:34:20 +0530 Subject: [PATCH 03/10] Fixed an NPE due to wrong var name --- .../com/gemstone/gemfire/internal/cache/TXBatchMessage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java index 0ea5d7de9..fc6fc4d1b 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java @@ -159,7 +159,7 @@ final void apply(final TXStateProxy tx) { final int numOps = this.pendingOps.size(); // take pendingTXRegionStates lock first so that // GII thread doesn't block on TXRegionState. - boolean[] locked = new boolean[pendingOpsRegion.size()]; + boolean[] locked = new boolean[pendingOpsRegions.size()]; int lockedIndex = 0; for (LocalRegion r : pendingOpsRegions) { if (!r.isInitialized()) { From d9f7e431c2090eabcccabc363cc937cf124fb423 Mon Sep 17 00:00:00 2001 From: Suranjan Kumar Date: Thu, 1 Feb 2018 16:37:06 +0530 Subject: [PATCH 04/10] Use StoppableReentrantReadWriteLock Instead of NonReentrantReadWriteLock, use StoppableReentrantReadWriteLock --- .../gemfire/internal/cache/ImageState.java | 4 -- .../gemfire/internal/cache/TXRegionState.java | 9 ++-- .../internal/cache/UnsharedImageState.java | 41 ++++++++----------- 3 files changed, 21 insertions(+), 33 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java index ccf34a46e..007bbbaab 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java @@ -188,8 +188,4 @@ public interface VersionTagEntry { public long getRegionVersion(); } - public boolean isPendingTXRegionStatesWriteLocked(); - - public Thread getPendingTXRegionStatesLockOwner(); - } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java index 08657f597..3a25d7d27 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java @@ -157,11 +157,9 @@ public TXRegionState(final LocalRegion r, final TXState tx) { // check if lock already taken by the same thread // in that case don't take lock ImageState imgState = r.getImageState(); - boolean alreadyLocked = (imgState.isPendingTXRegionStatesWriteLocked() - && imgState.getPendingTXRegionStatesLockOwner().getId() == Thread.currentThread().getId()); + if (!r.isInitialized() - && (alreadyLocked - || imgState.lockPendingTXRegionStates(true, false))) { + && imgState.lockPendingTXRegionStates(true, false)) { try { if (!r.getImageState().addPendingTXRegionState(this)) { this.pendingTXOps = null; @@ -171,8 +169,7 @@ public TXRegionState(final LocalRegion r, final TXState tx) { this.pendingTXLockFlags = new TIntArrayList(); } } finally { - if (!alreadyLocked) - r.getImageState().unlockPendingTXRegionStates(true); + r.getImageState().unlockPendingTXRegionStates(true); } } else { this.pendingTXOps = null; diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java index 208661606..44920b60c 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java @@ -21,6 +21,7 @@ import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem; import com.gemstone.gemfire.i18n.LogWriterI18n; import com.gemstone.gemfire.internal.Assert; +import com.gemstone.gemfire.internal.cache.locks.ReentrantReadWriteWriteShareLock; import com.gemstone.gemfire.internal.util.concurrent.StoppableNonReentrantLock; import com.gemstone.gemfire.internal.util.concurrent.StoppableReentrantReadWriteLock; import com.gemstone.gemfire.internal.cache.locks.NonReentrantReadWriteLock; @@ -76,7 +77,7 @@ public class UnsharedImageState implements ImageState { * GII is complete */ private volatile THashMapWithCreate pendingTXRegionStates; - private final NonReentrantReadWriteLock pendingTXRegionStatesLock; + private final StoppableReentrantReadWriteLock pendingTXRegionStatesLock; private volatile Thread pendingTXRegionStatesLockOwner; private final AtomicInteger pendingTXOrder; private volatile TObjectIntHashMap finishedTXIdOrders; @@ -102,7 +103,7 @@ public class UnsharedImageState implements ImageState { this.failedEvents = new ConcurrentTHashSet(2); this.pendingTXRegionStates = isLocal ? null : new THashMapWithCreate(); this.pendingTXRegionStatesLock = isLocal ? null - : new NonReentrantReadWriteLock(stopper); + : new StoppableReentrantReadWriteLock(stopper); this.pendingTXOrder = new AtomicInteger(0); } @@ -134,16 +135,6 @@ public void setRequestedUnappliedDelta(boolean flag) { requestedDelta = flag; } - @Override - public boolean isPendingTXRegionStatesWriteLocked() { - return this.pendingTXRegionStatesLock.isWriteLocked(); - } - - @Override - public Thread getPendingTXRegionStatesLockOwner() { - return pendingTXRegionStatesLockOwner; - } - @Override public boolean requestedUnappliedDelta() { return requestedDelta; @@ -413,7 +404,7 @@ public TXRegionState getPendingTXRegionState(TXId txId, boolean lock) { TXRegionState txrs = null; if (lock) { if (Thread.currentThread() != this.pendingTXRegionStatesLockOwner) { - this.pendingTXRegionStatesLock.attemptReadLock(-1); + this.pendingTXRegionStatesLock.readLock().lock(); } else { lock = false; @@ -432,7 +423,7 @@ public TXRegionState getPendingTXRegionState(TXId txId, boolean lock) { } } finally { if (lock) { - this.pendingTXRegionStatesLock.releaseReadLock(); + this.pendingTXRegionStatesLock.readLock().unlock(); } } return txrs; @@ -447,6 +438,10 @@ public boolean lockPendingTXRegionStates(final boolean forWrite, if (!force && this.pendingTXRegionStates == null) { return false; } + if (this.pendingTXRegionStatesLockOwner == Thread.currentThread()) { + return true; + } + if (TXStateProxy.LOG_FINE) { final LogWriterI18n logger = InternalDistributedSystem.getLoggerI18n(); if (logger != null) { @@ -456,11 +451,11 @@ public boolean lockPendingTXRegionStates(final boolean forWrite, } } if (forWrite) { - this.pendingTXRegionStatesLock.attemptWriteLock(-1); + this.pendingTXRegionStatesLock.writeLock().lock(); this.pendingTXRegionStatesLockOwner = Thread.currentThread(); } else { - this.pendingTXRegionStatesLock.attemptReadLock(-1); + this.pendingTXRegionStatesLock.readLock().lock(); } if (this.pendingTXRegionStates != null) { if (TXStateProxy.LOG_FINE) { @@ -476,10 +471,10 @@ public boolean lockPendingTXRegionStates(final boolean forWrite, else { if (forWrite) { this.pendingTXRegionStatesLockOwner = null; - this.pendingTXRegionStatesLock.releaseWriteLock(); + this.pendingTXRegionStatesLock.writeLock().unlock(); } else { - this.pendingTXRegionStatesLock.releaseReadLock(); + this.pendingTXRegionStatesLock.readLock().unlock(); } return false; } @@ -493,10 +488,10 @@ public void unlockPendingTXRegionStates(final boolean forWrite) { if (this.pendingTXRegionStatesLock != null) { if (forWrite) { this.pendingTXRegionStatesLockOwner = null; - this.pendingTXRegionStatesLock.releaseWriteLock(); + this.pendingTXRegionStatesLock.writeLock().unlock(); } else { - this.pendingTXRegionStatesLock.releaseReadLock(); + this.pendingTXRegionStatesLock.readLock().unlock(); } if (TXStateProxy.LOG_FINE) { final LogWriterI18n logger = InternalDistributedSystem.getLoggerI18n(); @@ -540,7 +535,7 @@ public void setTXOrderForFinish(TXRegionState txrs) { if (this.pendingTXRegionStatesLock != null) { TObjectIntHashMap finishedOrders; // assume read lock on pendingTXRegionStates is already held - Assert.assertTrue(this.pendingTXRegionStatesLock.numReaders() > 0); + // Assert.assertTrue(this.pendingTXRegionStatesLock.readLock(). > 0); if ((finishedOrders = this.finishedTXIdOrders) != null) { int order = finishedOrders.get(txrs.getTXState().getTransactionId()); if (order != 0) { @@ -574,7 +569,7 @@ public void mergeFinishedTXOrders(final LocalRegion region, final Collection txIds) { final THashMapWithCreate pendingTXRS = this.pendingTXRegionStates; if (pendingTXRS != null) { - this.pendingTXRegionStatesLock.attemptWriteLock(-1); + this.pendingTXRegionStatesLock.writeLock().lock(); try { // first get the ordering for finished transactions from TX manager; // this is deliberately invoked under the lock to sync against @@ -608,7 +603,7 @@ public void mergeFinishedTXOrders(final LocalRegion region, this.pendingTXOrder.addAndGet(increment); this.finishedTXIdOrders = txIdOrders; } finally { - this.pendingTXRegionStatesLock.releaseWriteLock(); + this.pendingTXRegionStatesLock.writeLock().unlock(); } } } From 2baa00580d4c3235dae4ef36426bf8852da5fe93 Mon Sep 17 00:00:00 2001 From: Suranjan Kumar Date: Thu, 1 Feb 2018 17:54:04 +0530 Subject: [PATCH 05/10] Simplified code to lock and unlock lockPendingTXRegionStates --- .../gemfire/internal/cache/TXBatchMessage.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java index fc6fc4d1b..c33509364 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java @@ -159,12 +159,10 @@ final void apply(final TXStateProxy tx) { final int numOps = this.pendingOps.size(); // take pendingTXRegionStates lock first so that // GII thread doesn't block on TXRegionState. - boolean[] locked = new boolean[pendingOpsRegions.size()]; - int lockedIndex = 0; + ArrayList lockedRegions = new ArrayList<>(); for (LocalRegion r : pendingOpsRegions) { - if (!r.isInitialized()) { - locked[lockedIndex] = r.getImageState().lockPendingTXRegionStates(true, false); - lockedIndex++; + if (!r.isInitialized() && r.getImageState().lockPendingTXRegionStates(true, false)) { + lockedRegions.add(r); } } try { @@ -189,10 +187,8 @@ final void apply(final TXStateProxy tx) { } } finally { int index = 0; - for (LocalRegion r : pendingOpsRegions) { - if (locked[index]) - r.getImageState().unlockPendingTXRegionStates(true); - index++; + for (LocalRegion r : lockedRegions) { + r.getImageState().unlockPendingTXRegionStates(true); } } } finally { From bd343607c9bd03bd588bd838c96efbf9355ed894 Mon Sep 17 00:00:00 2001 From: Suranjan Kumar Date: Thu, 1 Feb 2018 18:43:42 +0530 Subject: [PATCH 06/10] Missed files --- .../com/gemstone/gemfire/internal/cache/TXBatchMessage.java | 1 - .../gemstone/gemfire/internal/cache/UnsharedImageState.java | 4 ---- 2 files changed, 5 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java index c33509364..da934c5f9 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java @@ -186,7 +186,6 @@ final void apply(final TXStateProxy tx) { } } } finally { - int index = 0; for (LocalRegion r : lockedRegions) { r.getImageState().unlockPendingTXRegionStates(true); } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java index 44920b60c..1b5c56105 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java @@ -438,10 +438,6 @@ public boolean lockPendingTXRegionStates(final boolean forWrite, if (!force && this.pendingTXRegionStates == null) { return false; } - if (this.pendingTXRegionStatesLockOwner == Thread.currentThread()) { - return true; - } - if (TXStateProxy.LOG_FINE) { final LogWriterI18n logger = InternalDistributedSystem.getLoggerI18n(); if (logger != null) { From 9b61ae5e0ca546276c4056ba32d699eb4360ce70 Mon Sep 17 00:00:00 2001 From: Suranjan Kumar Date: Thu, 1 Feb 2018 20:06:02 +0530 Subject: [PATCH 07/10] Wwait for Initialization before taking lock In case of co-ordinator wait for initialization --- .../internal/cache/TXBatchMessage.java | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java index da934c5f9..f79832bcd 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java @@ -159,35 +159,42 @@ final void apply(final TXStateProxy tx) { final int numOps = this.pendingOps.size(); // take pendingTXRegionStates lock first so that // GII thread doesn't block on TXRegionState. - ArrayList lockedRegions = new ArrayList<>(); - for (LocalRegion r : pendingOpsRegions) { + //ArrayList lockedRegions = new ArrayList<>(); + /*for (LocalRegion r : pendingOpsRegions) { if (!r.isInitialized() && r.getImageState().lockPendingTXRegionStates(true, false)) { lockedRegions.add(r); } - } - try { - for (int index = 0; index < numOps; index++) { - entry = this.pendingOps.get(index); - if (pendingOpsRegion == null) { - region = this.pendingOpsRegions.get(index); - if (region.isUsedForPartitionedRegionBucket()) { - baseRegion = region.getPartitionedRegion(); - } else { - baseRegion = region; - } - } - if (txState.isCoordinator()) { - region.waitForData(); + }*/ + boolean lockedPendingTXregionState = false; + for (int index = 0; index < numOps; index++) { + entry = this.pendingOps.get(index); + if (pendingOpsRegion == null) { + region = this.pendingOpsRegions.get(index); + if (region.isUsedForPartitionedRegionBucket()) { + baseRegion = region.getPartitionedRegion(); + } else { + baseRegion = region; } + } + if (txState.isCoordinator()) { + region.waitForData(); + } + if (!region.isInitialized() && + region.getImageState().lockPendingTXRegionStates(true, false)) { + //lockedRegions.add(region); + lockedPendingTXregionState = true; + } + try { txrs = txState.writeRegion(region); if (txrs != null) { txState.applyPendingOperation(entry, lockFlags, txrs, region, baseRegion, eventTemplate, true, Boolean.TRUE, this); } - } - } finally { - for (LocalRegion r : lockedRegions) { - r.getImageState().unlockPendingTXRegionStates(true); + } finally { + //for (LocalRegion r : lockedRegions) { + if (lockedPendingTXregionState) { + region.getImageState().unlockPendingTXRegionStates(true); + } } } } finally { From dd37a65ab4c511230a50b23b02449c83dee1cfcb Mon Sep 17 00:00:00 2001 From: Suranjan Kumar Date: Thu, 1 Feb 2018 20:07:20 +0530 Subject: [PATCH 08/10] Reset the flag --- .../java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java | 1 + 1 file changed, 1 insertion(+) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java index f79832bcd..e71058d0a 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXBatchMessage.java @@ -194,6 +194,7 @@ final void apply(final TXStateProxy tx) { //for (LocalRegion r : lockedRegions) { if (lockedPendingTXregionState) { region.getImageState().unlockPendingTXRegionStates(true); + lockedPendingTXregionState = false; } } } From 66f00e430f4c9b065d696b910ef6cec7624a914b Mon Sep 17 00:00:00 2001 From: Suranjan Kumar Date: Fri, 2 Feb 2018 11:50:48 +0530 Subject: [PATCH 09/10] Revert NonReentrant to Reentrant lock In the TXRegionState constructor check if write lock already taken In that case don't try to take lock again. --- .../gemfire/internal/cache/ImageState.java | 2 ++ .../gemfire/internal/cache/TXRegionState.java | 6 ++-- .../internal/cache/UnsharedImageState.java | 34 +++++++++++-------- .../java/sql/sqlTx/thinClient/thinClientTx.bt | 32 +++++++++-------- 4 files changed, 42 insertions(+), 32 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java index 007bbbaab..c3092d07a 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java @@ -188,4 +188,6 @@ public interface VersionTagEntry { public long getRegionVersion(); } + public boolean isWriteLockedBySameThread(); + } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java index 3a25d7d27..a7a690ee4 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java @@ -158,8 +158,9 @@ public TXRegionState(final LocalRegion r, final TXState tx) { // in that case don't take lock ImageState imgState = r.getImageState(); + boolean alreadyLocked = imgState.isWriteLockedBySameThread(); if (!r.isInitialized() - && imgState.lockPendingTXRegionStates(true, false)) { + && (alreadyLocked || imgState.lockPendingTXRegionStates(true, false))) { try { if (!r.getImageState().addPendingTXRegionState(this)) { this.pendingTXOps = null; @@ -169,7 +170,8 @@ public TXRegionState(final LocalRegion r, final TXState tx) { this.pendingTXLockFlags = new TIntArrayList(); } } finally { - r.getImageState().unlockPendingTXRegionStates(true); + if (!alreadyLocked) + r.getImageState().unlockPendingTXRegionStates(true); } } else { this.pendingTXOps = null; diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java index 1b5c56105..debd3c888 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java @@ -21,7 +21,6 @@ import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem; import com.gemstone.gemfire.i18n.LogWriterI18n; import com.gemstone.gemfire.internal.Assert; -import com.gemstone.gemfire.internal.cache.locks.ReentrantReadWriteWriteShareLock; import com.gemstone.gemfire.internal.util.concurrent.StoppableNonReentrantLock; import com.gemstone.gemfire.internal.util.concurrent.StoppableReentrantReadWriteLock; import com.gemstone.gemfire.internal.cache.locks.NonReentrantReadWriteLock; @@ -38,7 +37,6 @@ import java.util.Iterator; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Used on distributed replicated regions to track GII and various state. @@ -77,7 +75,7 @@ public class UnsharedImageState implements ImageState { * GII is complete */ private volatile THashMapWithCreate pendingTXRegionStates; - private final StoppableReentrantReadWriteLock pendingTXRegionStatesLock; + private final NonReentrantReadWriteLock pendingTXRegionStatesLock; private volatile Thread pendingTXRegionStatesLockOwner; private final AtomicInteger pendingTXOrder; private volatile TObjectIntHashMap finishedTXIdOrders; @@ -103,7 +101,7 @@ public class UnsharedImageState implements ImageState { this.failedEvents = new ConcurrentTHashSet(2); this.pendingTXRegionStates = isLocal ? null : new THashMapWithCreate(); this.pendingTXRegionStatesLock = isLocal ? null - : new StoppableReentrantReadWriteLock(stopper); + : new NonReentrantReadWriteLock(stopper); this.pendingTXOrder = new AtomicInteger(0); } @@ -135,6 +133,12 @@ public void setRequestedUnappliedDelta(boolean flag) { requestedDelta = flag; } + @Override + public boolean isWriteLockedBySameThread() { + return this.pendingTXRegionStatesLock.isWriteLocked() && + (this.pendingTXRegionStatesLockOwner == Thread.currentThread()); + } + @Override public boolean requestedUnappliedDelta() { return requestedDelta; @@ -404,7 +408,7 @@ public TXRegionState getPendingTXRegionState(TXId txId, boolean lock) { TXRegionState txrs = null; if (lock) { if (Thread.currentThread() != this.pendingTXRegionStatesLockOwner) { - this.pendingTXRegionStatesLock.readLock().lock(); + this.pendingTXRegionStatesLock.attemptReadLock(-1); } else { lock = false; @@ -423,7 +427,7 @@ public TXRegionState getPendingTXRegionState(TXId txId, boolean lock) { } } finally { if (lock) { - this.pendingTXRegionStatesLock.readLock().unlock(); + this.pendingTXRegionStatesLock.releaseReadLock(); } } return txrs; @@ -447,11 +451,11 @@ public boolean lockPendingTXRegionStates(final boolean forWrite, } } if (forWrite) { - this.pendingTXRegionStatesLock.writeLock().lock(); + this.pendingTXRegionStatesLock.attemptWriteLock(-1); this.pendingTXRegionStatesLockOwner = Thread.currentThread(); } else { - this.pendingTXRegionStatesLock.readLock().lock(); + this.pendingTXRegionStatesLock.attemptReadLock(-1); } if (this.pendingTXRegionStates != null) { if (TXStateProxy.LOG_FINE) { @@ -467,10 +471,10 @@ public boolean lockPendingTXRegionStates(final boolean forWrite, else { if (forWrite) { this.pendingTXRegionStatesLockOwner = null; - this.pendingTXRegionStatesLock.writeLock().unlock(); + this.pendingTXRegionStatesLock.releaseWriteLock(); } else { - this.pendingTXRegionStatesLock.readLock().unlock(); + this.pendingTXRegionStatesLock.releaseReadLock(); } return false; } @@ -484,10 +488,10 @@ public void unlockPendingTXRegionStates(final boolean forWrite) { if (this.pendingTXRegionStatesLock != null) { if (forWrite) { this.pendingTXRegionStatesLockOwner = null; - this.pendingTXRegionStatesLock.writeLock().unlock(); + this.pendingTXRegionStatesLock.releaseWriteLock(); } else { - this.pendingTXRegionStatesLock.readLock().unlock(); + this.pendingTXRegionStatesLock.releaseReadLock(); } if (TXStateProxy.LOG_FINE) { final LogWriterI18n logger = InternalDistributedSystem.getLoggerI18n(); @@ -531,7 +535,7 @@ public void setTXOrderForFinish(TXRegionState txrs) { if (this.pendingTXRegionStatesLock != null) { TObjectIntHashMap finishedOrders; // assume read lock on pendingTXRegionStates is already held - // Assert.assertTrue(this.pendingTXRegionStatesLock.readLock(). > 0); + Assert.assertTrue(this.pendingTXRegionStatesLock.numReaders() > 0); if ((finishedOrders = this.finishedTXIdOrders) != null) { int order = finishedOrders.get(txrs.getTXState().getTransactionId()); if (order != 0) { @@ -565,7 +569,7 @@ public void mergeFinishedTXOrders(final LocalRegion region, final Collection txIds) { final THashMapWithCreate pendingTXRS = this.pendingTXRegionStates; if (pendingTXRS != null) { - this.pendingTXRegionStatesLock.writeLock().lock(); + this.pendingTXRegionStatesLock.attemptWriteLock(-1); try { // first get the ordering for finished transactions from TX manager; // this is deliberately invoked under the lock to sync against @@ -599,7 +603,7 @@ public void mergeFinishedTXOrders(final LocalRegion region, this.pendingTXOrder.addAndGet(increment); this.finishedTXIdOrders = txIdOrders; } finally { - this.pendingTXRegionStatesLock.writeLock().unlock(); + this.pendingTXRegionStatesLock.releaseWriteLock(); } } } diff --git a/tests/sql/src/main/java/sql/sqlTx/thinClient/thinClientTx.bt b/tests/sql/src/main/java/sql/sqlTx/thinClient/thinClientTx.bt index 680b5629b..b32d54916 100644 --- a/tests/sql/src/main/java/sql/sqlTx/thinClient/thinClientTx.bt +++ b/tests/sql/src/main/java/sql/sqlTx/thinClient/thinClientTx.bt @@ -1,3 +1,18 @@ +sql/sqlTx/thinClient/concUpdateTxClientHA.conf + locatorHosts = 1 locatorVMsPerHost =1 locatorThreadsPerVM = 1 + serverHosts=1 serverVMsPerHost=5 serverThreadsPerVM=1 + clientHosts = 1 clientVMsPerHost =6 clientThreadsPerVM = 5 + nobatching=true + redundantCopies = 1 + +sql/sqlTx/thinClient/concUpdateTxClientHA.conf + locatorHosts = 1 locatorVMsPerHost =1 locatorThreadsPerVM = 1 + serverHosts=1 serverVMsPerHost=5 serverThreadsPerVM=1 + clientHosts = 1 clientVMsPerHost =6 clientThreadsPerVM = 5 + nobatching=false + redundantCopies = 1 + + sql/sqlTx/thinClient/multiTablesTxClient.conf locatorHosts = 1 locatorVMsPerHost =1 locatorThreadsPerVM = 1 serverHosts=1 serverVMsPerHost=2, 5 serverThreadsPerVM=1 @@ -78,20 +93,7 @@ sql/sqlTx/thinClient/randomPartitionTablesIndexTxNoBatchingClientHA.conf workIterationsPerThread=500 redundantCopies = 1 -sql/sqlTx/thinClient/concUpdateTxClientHA.conf - locatorHosts = 1 locatorVMsPerHost =1 locatorThreadsPerVM = 1 - serverHosts=1 serverVMsPerHost=5 serverThreadsPerVM=1 - clientHosts = 1 clientVMsPerHost =6 clientThreadsPerVM = 5 - nobatching=true - redundantCopies = 1 - -sql/sqlTx/thinClient/concUpdateTxClientHA.conf - locatorHosts = 1 locatorVMsPerHost =1 locatorThreadsPerVM = 1 - serverHosts=1 serverVMsPerHost=5 serverThreadsPerVM=1 - clientHosts = 1 clientVMsPerHost =6 clientThreadsPerVM = 5 - nobatching=false - redundantCopies = 1 - + sql/sqlTx/thinClient/randomPartitionTablesIndexUniqTxClient.conf locatorHosts = 1 locatorVMsPerHost =1 locatorThreadsPerVM = 1 serverHosts=1 serverVMsPerHost=5 serverThreadsPerVM=1 @@ -138,4 +140,4 @@ sql/sqlTx/thinClient/randomPartitionNewTablesBatchingTxClientHA.conf clientHosts = 1 clientVMsPerHost =6 clientThreadsPerVM = 8 workIterationsPerThread=500 redundantCopies = 1, 2, 3 - \ No newline at end of file + From 9404c94aaf949b083cf7df0e88a324720397a9f6 Mon Sep 17 00:00:00 2001 From: suranjan kumar Date: Thu, 5 Jul 2018 10:54:49 +0530 Subject: [PATCH 10/10] Changed the lock to ReentrantReadWriteWriteShareLock --- .../gemfire/internal/cache/ImageState.java | 2 +- .../gemfire/internal/cache/TXRegionState.java | 6 +- .../internal/cache/UnsharedImageState.java | 64 ++++++++++++------- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java index c3092d07a..a86be9b7f 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java @@ -188,6 +188,6 @@ public interface VersionTagEntry { public long getRegionVersion(); } - public boolean isWriteLockedBySameThread(); + //public boolean isWriteLockedBySameThread(); } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java index 0329c8d0a..1d391a262 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java @@ -155,9 +155,9 @@ public TXRegionState(final LocalRegion r, final TXState tx) { // in that case don't take lock ImageState imgState = r.getImageState(); - boolean alreadyLocked = imgState.isWriteLockedBySameThread(); + //boolean alreadyLocked = imgState.isWriteLockedBySameThread(); if (!r.isInitialized() - && (alreadyLocked || imgState.lockPendingTXRegionStates(true, false))) { + && (imgState.lockPendingTXRegionStates(true, false))) { try { if (!r.getImageState().addPendingTXRegionState(this)) { this.pendingTXOps = null; @@ -167,7 +167,7 @@ public TXRegionState(final LocalRegion r, final TXState tx) { this.pendingTXLockFlags = new TIntArrayList(); } } finally { - if (!alreadyLocked) + //if (!alreadyLocked) r.getImageState().unlockPendingTXRegionStates(true); } } else { diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java index debd3c888..bd50fbd3e 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java @@ -21,6 +21,8 @@ import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem; import com.gemstone.gemfire.i18n.LogWriterI18n; import com.gemstone.gemfire.internal.Assert; +import com.gemstone.gemfire.internal.cache.locks.LockMode; +import com.gemstone.gemfire.internal.cache.locks.ReentrantReadWriteWriteShareLock; import com.gemstone.gemfire.internal.util.concurrent.StoppableNonReentrantLock; import com.gemstone.gemfire.internal.util.concurrent.StoppableReentrantReadWriteLock; import com.gemstone.gemfire.internal.cache.locks.NonReentrantReadWriteLock; @@ -75,8 +77,10 @@ public class UnsharedImageState implements ImageState { * GII is complete */ private volatile THashMapWithCreate pendingTXRegionStates; - private final NonReentrantReadWriteLock pendingTXRegionStatesLock; - private volatile Thread pendingTXRegionStatesLockOwner; + private final ReentrantReadWriteWriteShareLock pendingTXRegionStatesLock; + + + //private volatile Thread pendingTXRegionStatesLockOwner; private final AtomicInteger pendingTXOrder; private volatile TObjectIntHashMap finishedTXIdOrders; @@ -101,7 +105,7 @@ public class UnsharedImageState implements ImageState { this.failedEvents = new ConcurrentTHashSet(2); this.pendingTXRegionStates = isLocal ? null : new THashMapWithCreate(); this.pendingTXRegionStatesLock = isLocal ? null - : new NonReentrantReadWriteLock(stopper); + : new ReentrantReadWriteWriteShareLock(); this.pendingTXOrder = new AtomicInteger(0); } @@ -133,11 +137,11 @@ public void setRequestedUnappliedDelta(boolean flag) { requestedDelta = flag; } - @Override + /*@Override public boolean isWriteLockedBySameThread() { return this.pendingTXRegionStatesLock.isWriteLocked() && (this.pendingTXRegionStatesLockOwner == Thread.currentThread()); - } + }*/ @Override public boolean requestedUnappliedDelta() { @@ -407,12 +411,13 @@ public TXRegionState getPendingTXRegionState(TXId txId, boolean lock) { } TXRegionState txrs = null; if (lock) { - if (Thread.currentThread() != this.pendingTXRegionStatesLockOwner) { - this.pendingTXRegionStatesLock.attemptReadLock(-1); - } - else { + //if (Thread.currentThread() != this.pendingTXRegionStatesLockOwner) { + this.pendingTXRegionStatesLock.attemptLock(LockMode.SH,-1, this); + //this.pendingTXRegionStatesLock.attemptReadLock(-1); + //} + //else { lock = false; - } + //} } try { if (this.pendingTXRegionStates != null) { @@ -427,7 +432,8 @@ public TXRegionState getPendingTXRegionState(TXId txId, boolean lock) { } } finally { if (lock) { - this.pendingTXRegionStatesLock.releaseReadLock(); + this.pendingTXRegionStatesLock.releaseLock(LockMode.SH,false, this); + //this.pendingTXRegionStatesLock.releaseReadLock(); } } return txrs; @@ -451,11 +457,13 @@ public boolean lockPendingTXRegionStates(final boolean forWrite, } } if (forWrite) { - this.pendingTXRegionStatesLock.attemptWriteLock(-1); - this.pendingTXRegionStatesLockOwner = Thread.currentThread(); + this.pendingTXRegionStatesLock.attemptLock(LockMode.EX,-1, this); + //this.pendingTXRegionStatesLock.attemptWriteLock(-1); + //this.pendingTXRegionStatesLockOwner = Thread.currentThread(); } else { - this.pendingTXRegionStatesLock.attemptReadLock(-1); + this.pendingTXRegionStatesLock.attemptLock(LockMode.SH,-1, this); + //this.pendingTXRegionStatesLock.attemptReadLock(-1); } if (this.pendingTXRegionStates != null) { if (TXStateProxy.LOG_FINE) { @@ -470,11 +478,14 @@ public boolean lockPendingTXRegionStates(final boolean forWrite, } else { if (forWrite) { - this.pendingTXRegionStatesLockOwner = null; - this.pendingTXRegionStatesLock.releaseWriteLock(); + //this.pendingTXRegionStatesLockOwner = null; + //this.pendingTXRegionStatesLock.releaseWriteLock(); + this.pendingTXRegionStatesLock.releaseLock(LockMode.EX, false, this); } else { - this.pendingTXRegionStatesLock.releaseReadLock(); + this.pendingTXRegionStatesLock.releaseLock(LockMode.SH, false, this); + + //this.pendingTXRegionStatesLock.releaseReadLock(); } return false; } @@ -487,11 +498,15 @@ public boolean lockPendingTXRegionStates(final boolean forWrite, public void unlockPendingTXRegionStates(final boolean forWrite) { if (this.pendingTXRegionStatesLock != null) { if (forWrite) { - this.pendingTXRegionStatesLockOwner = null; - this.pendingTXRegionStatesLock.releaseWriteLock(); + //this.pendingTXRegionStatesLockOwner = null; + //this.pendingTXRegionStatesLock.releaseWriteLock(); + this.pendingTXRegionStatesLock.releaseLock(LockMode.EX, false, this); + } else { - this.pendingTXRegionStatesLock.releaseReadLock(); + //this.pendingTXRegionStatesLock.releaseReadLock(); + this.pendingTXRegionStatesLock.releaseLock(LockMode.SH, false, this); + } if (TXStateProxy.LOG_FINE) { final LogWriterI18n logger = InternalDistributedSystem.getLoggerI18n(); @@ -535,7 +550,8 @@ public void setTXOrderForFinish(TXRegionState txrs) { if (this.pendingTXRegionStatesLock != null) { TObjectIntHashMap finishedOrders; // assume read lock on pendingTXRegionStates is already held - Assert.assertTrue(this.pendingTXRegionStatesLock.numReaders() > 0); + //Assert.assertTrue(this.pendingTXRegionStatesLock.numReaders() > 0); + Assert.assertTrue(this.pendingTXRegionStatesLock.numReadOnlyLocks() > 0); if ((finishedOrders = this.finishedTXIdOrders) != null) { int order = finishedOrders.get(txrs.getTXState().getTransactionId()); if (order != 0) { @@ -569,7 +585,8 @@ public void mergeFinishedTXOrders(final LocalRegion region, final Collection txIds) { final THashMapWithCreate pendingTXRS = this.pendingTXRegionStates; if (pendingTXRS != null) { - this.pendingTXRegionStatesLock.attemptWriteLock(-1); + //this.pendingTXRegionStatesLock.attemptWriteLock(-1); + this.pendingTXRegionStatesLock.attemptLock(LockMode.EX, -1, this); try { // first get the ordering for finished transactions from TX manager; // this is deliberately invoked under the lock to sync against @@ -603,7 +620,8 @@ public void mergeFinishedTXOrders(final LocalRegion region, this.pendingTXOrder.addAndGet(increment); this.finishedTXIdOrders = txIdOrders; } finally { - this.pendingTXRegionStatesLock.releaseWriteLock(); + //this.pendingTXRegionStatesLock.releaseWriteLock(); + this.pendingTXRegionStatesLock.releaseLock(LockMode.EX, false, this); } } }