From 7fd60f018074f6fd4e2c5ac0b834bcfed8061cfa Mon Sep 17 00:00:00 2001 From: anastas Date: Tue, 22 Mar 2016 13:23:22 +0200 Subject: [PATCH 1/9] My squashed commits Signed-off-by: anastas --- .../hadoop/hbase/regionserver/CellBlock.java | 467 ++++++++++++++++++ .../regionserver/CellBlockObjectArray.java | 52 ++ .../regionserver/CellBlockSerialized.java | 85 ++++ .../hbase/regionserver/HeapMemStoreLAB.java | 63 ++- .../hbase/regionserver/MemStoreChunkPool.java | 45 +- .../hbase/regionserver/TestCellBlocksSet.java | 174 +++++++ 6 files changed, 872 insertions(+), 14 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlock.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockObjectArray.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockSerialized.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlock.java new file mode 100644 index 000000000000..6403a5266f1f --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlock.java @@ -0,0 +1,467 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Cellersion 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY CellIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.Cell; + +import java.util.Collection; +import java.util.Comparator; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Set; + + + +/** + * CellBlock stores a constant number of elements and is immutable after creation stage. + * Due to being immutable the CellBlock can be implemented as array. + * The actual array is on- or off-heap and is implemented in concrete class derived from CellBlock. + * The CellBlock uses no synchronization primitives, it is assumed to be created by a + * single thread and then it can be read-only by multiple threads. + */ +public abstract class CellBlock implements ConcurrentNavigableMap { + + private final Comparator comparator; + private int minCellIdx = 0; // the index of the minimal cell (for sub-sets) + private int maxCellIdx = 0; // the index of the maximal cell (for sub-sets) + private boolean descending = false; + + /* C-tor */ + public CellBlock(Comparator comparator, int min, int max, boolean d){ + this.comparator = comparator; + this.minCellIdx = min; + this.maxCellIdx = max; + this.descending = d; + } + + /* Used for abstract CellBlock creation, implemented by derived class */ + protected abstract CellBlock createCellBlocks(Comparator comparator, int min, + int max, boolean descending); + + /* Returns the i-th cell in the cell block */ + protected abstract Cell getCellFromIndex(int i); + + /** + * Binary search for a given key in between given boundaries of the array. + * Positive returned numbers mean the index. + * Negative returned numbers means the key not found. + * The absolute value of the output is the + * possible insert index for the searched key: (-1 * insertion point) - 1 + * @param needle The key to look for in all of the entries + * @return Same return value as Arrays.binarySearch. + */ + private int find(Cell needle) { + int begin = minCellIdx; + int end = maxCellIdx - 1; + + while (begin <= end) { + int mid = begin + ((end - begin) / 2); + + Cell midCell = getCellFromIndex(mid); + + int compareRes = comparator.compare(midCell, needle); + + // 0 means equals. We found the key. + if (compareRes == 0) return mid; + else if (compareRes < 0) { + // midCell is less than needle so we need to look at farther up + begin = mid + 1; + } else { + // midCell is greater than needle so we need to look down + end = mid - 1; + } + } + + return (-1 * begin) - 1; + } + + private int getValidIndex(Cell key, boolean inclusive) { + int index = find(key); + if (inclusive && index >= 0) index++; + else if (index < 0) index = -(index + 1) - 1; + return index; + } + + @Override + public Comparator comparator() { + return comparator; + } + + @Override + public int size() { + return maxCellIdx-minCellIdx; + } + + @Override + public boolean isEmpty() { + return (maxCellIdx==minCellIdx); + } + + + // ---------------- Sub-Maps ---------------- + @Override + public ConcurrentNavigableMap subMap( Cell fromKey, + boolean fromInclusive, + Cell toKey, + boolean toInclusive) { + int toIndex = getValidIndex(toKey, toInclusive); + int fromIndex = (getValidIndex(fromKey, !fromInclusive)); + + if (fromIndex > toIndex) throw new IllegalArgumentException("inconsistent range"); + return createCellBlocks(comparator, fromIndex, toIndex, descending); + } + + @Override + public ConcurrentNavigableMap headMap(Cell toKey, boolean inclusive) { + int index = getValidIndex(toKey, inclusive); + return createCellBlocks(comparator, minCellIdx, index, descending); + } + + @Override + public ConcurrentNavigableMap tailMap(Cell fromKey, boolean inclusive) { + int index = (getValidIndex(fromKey, !inclusive)); + return createCellBlocks(comparator, index, maxCellIdx, descending); + } + + @Override + public ConcurrentNavigableMap descendingMap() { + return createCellBlocks(comparator, minCellIdx, maxCellIdx, true); + } + + @Override + public ConcurrentNavigableMap subMap(Cell k1, Cell k2) { + return this.subMap(k1, true, k2, true); + } + + @Override + public ConcurrentNavigableMap headMap(Cell k) { + return this.headMap(k, true); + } + + @Override + public ConcurrentNavigableMap tailMap(Cell k) { + return this.tailMap(k, true); + } + + + // -------------------------------- Key's getters -------------------------------- + @Override + public Cell firstKey() { + if (isEmpty()) return null; + if (descending) getCellFromIndex(maxCellIdx-1); + return getCellFromIndex(minCellIdx); + } + + @Override + public Cell lastKey() { + if (isEmpty()) return null; + if (descending) return getCellFromIndex(minCellIdx); + return getCellFromIndex(maxCellIdx-1); + } + + @Override + public Cell lowerKey(Cell k) { + if (isEmpty()) return null; + int index = find(k); + if (descending) { + if (index >= 0) index++; // There's a key exactly equal. + else index = -(index + 1); + } else { + if (index >= 0) index--; // There's a key exactly equal. + else index = -(index + 1) - 1; + } + return (index < minCellIdx || index >= maxCellIdx) ? null : getCellFromIndex(index); + } + + @Override + public Cell floorKey(Cell k) { + if (isEmpty()) return null; + int index = find(k); + if (descending) { + if (index < 0) index = -(index + 1); + } else { + if (index < 0) index = -(index + 1) - 1; + } + return (index < minCellIdx || index >= maxCellIdx) ? null : getCellFromIndex(index); + } + + @Override + public Cell ceilingKey(Cell k) { + if (isEmpty()) return null; + int index = find(k); + if (descending) { + if (index < 0) index = -(index + 1) - 1; + } else { + if (index < 0) index = -(index + 1); + } + return (index < minCellIdx || index >= maxCellIdx) ? null : getCellFromIndex(index); + } + + @Override + public Cell higherKey(Cell k) { + if (isEmpty()) return null; + int index = find(k); + if (descending) { + if (index >= 0) index--; // There's a key exactly equal. + else index = -(index + 1) - 1; + } else { + if (index >= 0) index++; // There's a key exactly equal. + else index = -(index + 1); + } + return (index < minCellIdx || index >= maxCellIdx) ? null : getCellFromIndex(index); + } + + @Override + public boolean containsKey(Object o) { + int index = find((Cell) o); + return (index >= 0); + } + + @Override + public boolean containsValue(Object o) { // use containsKey(Object o) instead + throw new UnsupportedOperationException(); + } + + @Override + public Cell get(Object o) { + int index = find((Cell) o); + if (index >= 0) { + return getCellFromIndex(index); + } + return null; + } + + // -------------------------------- Entry's getters -------------------------------- + // all interfaces returning Entries are unsupported because we are dealing only with the keys + @Override + public Entry lowerEntry(Cell k) { + throw new UnsupportedOperationException(); + } + + @Override + public Entry higherEntry(Cell k) { + throw new UnsupportedOperationException(); + } + + @Override + public Entry ceilingEntry(Cell k) { + throw new UnsupportedOperationException(); + } + + @Override + public Entry floorEntry(Cell k) { + throw new UnsupportedOperationException(); + } + + @Override + public Entry firstEntry() { + throw new UnsupportedOperationException(); + } + + @Override + public Entry lastEntry() { + throw new UnsupportedOperationException(); + } + + @Override + public Entry pollFirstEntry() { + throw new UnsupportedOperationException(); + } + + @Override + public Entry pollLastEntry() { + throw new UnsupportedOperationException(); + } + + + // -------------------------------- Updates -------------------------------- + // All updating methods below are unsupported. + // Assuming an array of Cells will be allocated externally, + // fill up with Cells and provided in construction time. + // Later the structure is immutable. + @Override + public Cell put(Cell k, Cell v) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + @Override + public Cell remove(Object o) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean replace(Cell k, Cell v, Cell v1) { + throw new UnsupportedOperationException(); + } + + @Override + public void putAll(Map map) { + throw new UnsupportedOperationException(); + } + + @Override + public Cell putIfAbsent(Cell k, Cell v) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean remove(Object o, Object o1) { + throw new UnsupportedOperationException(); + } + + @Override + public Cell replace(Cell k, Cell v) { + throw new UnsupportedOperationException(); + } + + + // -------------------------------- Sub-Sets -------------------------------- + @Override + public NavigableSet navigableKeySet() { + throw new UnsupportedOperationException(); + } + + @Override + public NavigableSet descendingKeySet() { + throw new UnsupportedOperationException(); + } + + @Override + public NavigableSet keySet() { + throw new UnsupportedOperationException(); + } + + @Override + public Collection values() { + return new CellBlocksCollection(); + } + + @Override + public Set> entrySet() { + throw new UnsupportedOperationException(); + } + + + // -------------------------------- Iterator K -------------------------------- + private final class CellBlocksIterator implements Iterator { + int index; + + private CellBlocksIterator() { + index = descending ? maxCellIdx-1 : minCellIdx; + } + + @Override + public boolean hasNext() { + return descending ? (index >= minCellIdx) : (index < maxCellIdx); + } + + @Override + public Cell next() { + Cell result = getCellFromIndex(index); + if (descending) index--; + else index++; + return result; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } + + + // -------------------------------- Collection -------------------------------- + private final class CellBlocksCollection implements Collection { + + @Override + public int size() { + return CellBlock.this.size(); + } + + @Override + public boolean isEmpty() { + return CellBlock.this.isEmpty(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean contains(Object o) { + return containsKey(o); + } + + @Override + public Iterator iterator() { + return new CellBlocksIterator(); + } + + @Override + public Object[] toArray() { + throw new UnsupportedOperationException(); + } + + @Override + public T[] toArray(T[] ts) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean add(Cell k) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean remove(Object o) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean containsAll(Collection collection) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean addAll(Collection collection) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean removeAll(Collection collection) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean retainAll(Collection collection) { + throw new UnsupportedOperationException(); + } + + + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockObjectArray.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockObjectArray.java new file mode 100644 index 000000000000..bebdd7aee5f7 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockObjectArray.java @@ -0,0 +1,52 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Cellersion 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY CellIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.util.Comparator; +import org.apache.hadoop.hbase.Cell; + +/** + * CellBlockObjectArray is a simple array of Cells allocated using JVM. + * As all java arrays it is array of references pointing to Cell objects + */ +public class CellBlockObjectArray extends CellBlock { + + Cell[] block; + + /* The Cells Array is created only when CellBlockObjectArray is created, all sub-CellBlocks use + * boundary indexes */ + public CellBlockObjectArray(Comparator comparator, Cell[] b, int min, int max, + boolean d) { + super(comparator,min,max,d); + this.block = b; + } + + /* To be used by base class only to create a sub-CellBlock */ + @Override + protected CellBlock createCellBlocks(Comparator comparator, + int min, int max, boolean d) { + return new CellBlockObjectArray(comparator,this.block,min,max,d); + } + + @Override + protected Cell getCellFromIndex(int i) { + return block[i]; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockSerialized.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockSerialized.java new file mode 100644 index 000000000000..003cb952db66 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockSerialized.java @@ -0,0 +1,85 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Cellersion 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY CellIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.util.Comparator; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + + + +/** + * CellBlockSerialized is a byte array holding all that is needed to access a Cell, which + * is actually saved on another deeper byte array. + * Per Cell we have a reference to this deeper byte array B, offset in bytes in B (integer), + * and length in bytes in B (integer). In order to save reference to byte array we use the Chunk's + * indexes given by MSLAB (also integer). + * + * The B memory layout: + * + * <----------------- first Cell ---------------------> <-------------- second Cell + * ------------------------------------------------------------------------------------- ... + * | integer = x bytes | integer = x bytes | integer = x bytes | integer = x bytes | + * | reference to B | offset in B where | length of Cell's | reference to may be| ... + * | holding Cell data | Cell's data starts| data in B | another byte array | + * ------------------------------------------------------------------------------------- ... + */ +public class CellBlockSerialized extends CellBlock { + + private HeapMemStoreLAB.Chunk[] chunks; + private final HeapMemStoreLAB memStoreLAB; + private int numOfCellsInsideChunk; + private static final int BYTES_IN_CELL = 3*(Integer.SIZE / Byte.SIZE); // each Cell requires 3 integers + + public CellBlockSerialized(Comparator comparator, HeapMemStoreLAB memStoreLAB, + HeapMemStoreLAB.Chunk[] chunks, int min, int max, int chunkSize, boolean d) { + super(comparator,min,max, d); + this.chunks = chunks; + this.memStoreLAB = memStoreLAB; + this.numOfCellsInsideChunk = chunkSize / BYTES_IN_CELL; + } + + /* To be used by base class only to create a sub-CellBlock */ + @Override + protected CellBlock createCellBlocks(Comparator comparator, + int min, int max, boolean d) { + return new CellBlockSerialized(comparator, this.memStoreLAB, this.chunks, min, max, + this.numOfCellsInsideChunk* BYTES_IN_CELL, d); + } + + @Override + protected Cell getCellFromIndex(int i) { + // find correct chunk + int chunkIndex = (i / numOfCellsInsideChunk); + byte[] block = chunks[chunkIndex].getData(); + i = i - chunkIndex*numOfCellsInsideChunk; + + // find inside chunk + int offsetInBytes = i* BYTES_IN_CELL; + int chunkId = Bytes.toInt(block,offsetInBytes); + int offsetOfCell = Bytes.toInt(block,offsetInBytes+(Integer.SIZE / Byte.SIZE)); + int lengthOfCell = Bytes.toInt(block,offsetInBytes+2*(Integer.SIZE / Byte.SIZE)); + byte[] chunk = memStoreLAB.translateIdToChunk(chunkId).getData(); + + Cell result = new KeyValue(chunk, offsetOfCell, lengthOfCell); + return result; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java index f22a6e5c1ac8..228616bade7e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java @@ -31,6 +31,7 @@ import com.google.common.base.Preconditions; + /** * A memstore-local allocation buffer. *

@@ -88,9 +89,8 @@ public HeapMemStoreLAB(Configuration conf) { this.chunkPool = MemStoreChunkPool.getPool(conf); // if we don't exclude allocations >CHUNK_SIZE, we'd infiniteloop on one! - Preconditions.checkArgument( - maxAlloc <= chunkSize, - MAX_ALLOC_KEY + " must be less than " + CHUNK_SIZE_KEY); + Preconditions.checkArgument(maxAlloc <= chunkSize, + MAX_ALLOC_KEY + " must be less than " + CHUNK_SIZE_KEY); } /** @@ -112,7 +112,7 @@ public ByteRange allocateBytes(int size) { while (true) { Chunk c = getOrMakeChunk(); - // Try to allocate from this chunk + // Try to allocate from this chunk int allocOffset = c.alloc(size); if (allocOffset != -1) { // We succeeded - this is the common case - small alloc @@ -181,6 +181,7 @@ private void tryRetireChunk(Chunk c) { * allocate a new one from the JVM. */ private Chunk getOrMakeChunk() { + while (true) { // Try to get the chunk Chunk c = curChunk.get(); @@ -191,11 +192,21 @@ private Chunk getOrMakeChunk() { // No current chunk, so we want to allocate one. We race // against other allocators to CAS in an uninitialized chunk // (which is cheap to allocate) - c = (chunkPool != null) ? chunkPool.getChunk() : new Chunk(chunkSize); + + //c = (chunkPool != null) ? chunkPool.getChunk() : new Chunk(chunkSize, 5); //14921 + + if(chunkPool != null) { + c = chunkPool.getChunk(); + } else { + c = new Chunk(chunkSize, 5); + c.init(); + } + if (curChunk.compareAndSet(null, c)) { // we won race - now we need to actually do the expensive // allocation step - c.init(); + + //c.init(); //14921 this.chunkQueue.add(c); return c; } else if (chunkPool != null) { @@ -206,6 +217,24 @@ private Chunk getOrMakeChunk() { } } + /** 14921 + * Given a chunk ID return reference to the relevant chunk + * @return a chunk + */ + Chunk translateIdToChunk(int id) { + return chunkPool.translateIdToChunk(id); + } + + /** 14921 + * Use instead of allocateBytes() when new full chunk is needed + * @return a chunk + */ + Chunk allocateChunk() { + Chunk c = chunkPool.getChunk(); + this.chunkQueue.add(c); + return c; + } + /** * A chunk of memory out of which allocations are sliced. */ @@ -227,12 +256,18 @@ static class Chunk { /** Size of chunk in bytes */ private final int size; + /* 14921: A unique identifier of a chunk inside MemStoreChunkPool */ + private final int id; + + /* Chunk's index serves as replacement for pointer */ + /** * Create an uninitialized chunk. Note that memory is not allocated yet, so * this is cheap. * @param size in bytes */ - Chunk(int size) { + Chunk(int size, int id) { + this.id = id; this.size = size; } @@ -252,13 +287,15 @@ public void init() { assert failInit; // should be true. throw e; } + // Mark that it's ready for use boolean initted = nextFreeOffset.compareAndSet( UNINITIALIZED, 0); // We should always succeed the above CAS since only one thread // calls init()! - Preconditions.checkState(initted, - "Multiple threads tried to init same chunk"); + Preconditions.checkState(initted, "Multiple threads tried to init same chunk"); + + //org.junit.Assert.assertTrue("\n\n inside chunk initialization 3", false); } /** @@ -311,5 +348,13 @@ public String toString() { " allocs=" + allocCount.get() + "waste=" + (data.length - nextFreeOffset.get()); } + + public int getId() { + return id; + } // 14921 + + public byte[] getData() { + return data; + } // 14921 } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java index 628506059eef..bc85bc0e95ee 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java @@ -19,12 +19,16 @@ package org.apache.hadoop.hbase.regionserver; import java.lang.management.ManagementFactory; +//import java.util.concurrent.*; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -76,17 +80,21 @@ public class MemStoreChunkPool { private static final int statThreadPeriod = 60 * 5; private AtomicLong createdChunkCount = new AtomicLong(); private AtomicLong reusedChunkCount = new AtomicLong(); + private AtomicInteger chunkIDs = new AtomicInteger(1); // 14921 + + // 14921: IDs Mapping of all chunks (key 0 is forbidden) + private final ConcurrentMap chunksMap = new ConcurrentHashMap(); MemStoreChunkPool(Configuration conf, int chunkSize, int maxCount, int initialCount) { this.maxCount = maxCount; this.chunkSize = chunkSize; this.reclaimedChunks = new LinkedBlockingQueue(); + for (int i = 0; i < initialCount; i++) { - Chunk chunk = new Chunk(chunkSize); - chunk.init(); - reclaimedChunks.add(chunk); + Chunk chunk = allocateChunk(); } + final String n = Thread.currentThread().getName(); scheduleThreadPool = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat(n+"-MemStoreChunkPool Statistics") @@ -103,10 +111,11 @@ public class MemStoreChunkPool { Chunk getChunk() { Chunk chunk = reclaimedChunks.poll(); if (chunk == null) { - chunk = new Chunk(chunkSize); + chunk = allocateChunk(); createdChunkCount.incrementAndGet(); } else { chunk.reset(); + chunk.init(); // 14921 reusedChunkCount.incrementAndGet(); } return chunk; @@ -125,6 +134,14 @@ void putbackChunks(BlockingQueue chunks) { chunks.drainTo(reclaimedChunks, maxNumToPutback); } + /** + * Given a chunk ID return reference to the relevant chunk + * @return a chunk + */ + Chunk translateIdToChunk(int id) { + return chunksMap.get(id); + } + /** * Add the chunk to the pool, if the pool has achieved the max size, it will * skip it @@ -148,6 +165,24 @@ void clearChunks() { this.reclaimedChunks.clear(); } + /* + * Only used in testing + */ + ConcurrentMap getChunksMap() { + return this.chunksMap; + } + + /* + * Allocate and register Chunk + */ + private Chunk allocateChunk() { + int newId = chunkIDs.getAndAdd(1); // the number of the new chunk + Chunk chunk = new Chunk(chunkSize,newId); + chunksMap.put(newId, chunk); + chunk.init(); + return chunk; + } + private static class StatisticsThread extends Thread { MemStoreChunkPool mcp; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java new file mode 100644 index 000000000000..1a368df9ea96 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java @@ -0,0 +1,174 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import junit.framework.TestCase; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +import java.util.Iterator; +import java.util.SortedSet; +import static org.junit.Assert.assertTrue; + +@Category({RegionServerTests.class, SmallTests.class}) +public class TestCellBlocksSet extends TestCase { + + private static final int NUM_OF_CELLS = 3; + + private Cell cells[]; + private CellBlockObjectArray cbOnHeap; + private CellBlockSerialized cbOffHeap; + + private final static Configuration conf = new Configuration(); + private HeapMemStoreLAB mslab; + + + + protected void setUp() throws Exception { + super.setUp(); + + // create array of Cells to bass to the CellBlock under CellSet + final byte[] one = Bytes.toBytes(1); + final byte[] two = Bytes.toBytes(2); + final byte[] three = Bytes.toBytes(3); + final byte[] f = Bytes.toBytes("f"); + final byte[] q = Bytes.toBytes("q"); + final byte[] v = Bytes.toBytes(4); + + final KeyValue kv1 = new KeyValue(one, f, q, 10, v); + final KeyValue kv2 = new KeyValue(two, f, q, 20, v); + final KeyValue kv3 = new KeyValue(three, f, q, 30, v); + + cells = new Cell[] {kv1,kv2,kv3}; + cbOnHeap = new CellBlockObjectArray(CellComparator.COMPARATOR,cells,0,NUM_OF_CELLS,false); + + conf.setBoolean(SegmentFactory.USEMSLAB_KEY, true); + conf.setFloat(MemStoreChunkPool.CHUNK_POOL_MAXSIZE_KEY, 0.2f); + MemStoreChunkPool.chunkPoolDisabled = false; + mslab = new HeapMemStoreLAB(conf); + + HeapMemStoreLAB.Chunk[] c = shallowCellsToBuffer(kv1, kv2, kv3); + int chunkSize = conf.getInt(HeapMemStoreLAB.CHUNK_SIZE_KEY, HeapMemStoreLAB.CHUNK_SIZE_DEFAULT); + cbOffHeap = new CellBlockSerialized(CellComparator.COMPARATOR, mslab, + c, 0, NUM_OF_CELLS, chunkSize, false); + } + + /* Create and test CellSet based on CellBlockObjectArray */ + public void testCellBlocksOnHeap() throws Exception { + CellSet cs = new CellSet(cbOnHeap); + testCellBlocks(cs); + testIterators(cs); + } + + /* Create and test CellSet based on CellBlockSerialized */ + public void testCellBlocksOffHeap() throws Exception { + CellSet cs = new CellSet(cbOffHeap); + testCellBlocks(cs); + testIterators(cs); + } + + /* Generic basic test for immutable CellSet */ + private void testCellBlocks(CellSet cs) throws Exception { + assertEquals(NUM_OF_CELLS, cs.size()); // check size + + assertTrue(cs.contains(cells[0])); // check existance of the first + Cell first = cs.first(); + assertTrue(cells[0].equals(first)); + + assertTrue(cs.contains(cells[NUM_OF_CELLS - 1])); // check last + Cell last = cs.last(); + assertTrue(cells[NUM_OF_CELLS - 1].equals(last)); + + SortedSet tail = cs.tailSet(cells[1]); // check tail abd head sizes + assertEquals(2, tail.size()); + SortedSet head = cs.headSet(cells[1]); + assertEquals(1, head.size()); + + Cell tailFirst = tail.first(); + assertTrue(cells[1].equals(tailFirst)); + Cell tailLast = tail.last(); + assertTrue(cells[2].equals(tailLast)); + + Cell headFirst = head.first(); + assertTrue(cells[0].equals(headFirst)); + Cell headLast = head.last(); + assertTrue(cells[0].equals(headLast)); + } + + /* Generic iterators test for immutable CellSet */ + private void testIterators(CellSet cs) throws Exception { + + // Assert that we have NUM_OF_CELLS values and that they are in order + int count = 0; + for (Cell kv: cs) { + assertEquals("\n\n-------------------------------------------------------------------\n" + + "Comparing iteration number " + (count + 1) + " the returned cell: " + kv + + ", the first Cell in the CellBlocksMap: " + cells[count] + + ", and the same transformed to String: " + cells[count].toString() + + "\n-------------------------------------------------------------------\n", + cells[count], kv); + count++; + } + assertEquals(NUM_OF_CELLS, count); + + // Test descending iterator + count = 0; + for (Iterator i = cs.descendingIterator(); i.hasNext();) { + Cell kv = i.next(); + assertEquals(cells[NUM_OF_CELLS - (count + 1)], kv); + count++; + } + assertEquals(NUM_OF_CELLS, count); + } + + /* Create byte array holding shallow Cells referencing to the deep Cells data */ + private HeapMemStoreLAB.Chunk[] shallowCellsToBuffer(Cell kv1, Cell kv2, Cell kv3) { + HeapMemStoreLAB.Chunk chunkD = mslab.allocateChunk(); + HeapMemStoreLAB.Chunk chunkS = mslab.allocateChunk(); + HeapMemStoreLAB.Chunk result[] = {chunkS}; + + byte[] deepBuffer = chunkD.getData(); + byte[] shallowBuffer = chunkS.getData(); + int offset = 0; + int pos = offset; + KeyValueUtil.appendToByteArray(kv1, deepBuffer, offset); // write deep cell data + + pos = Bytes.putInt(shallowBuffer, pos, chunkD.getId()); // write deep chunk index + pos = Bytes.putInt(shallowBuffer, pos, offset); // offset + pos = Bytes.putInt(shallowBuffer, pos, KeyValueUtil.length(kv1)); // length + offset += KeyValueUtil.length(kv1); + + KeyValueUtil.appendToByteArray(kv2, deepBuffer, offset); // write deep cell data + pos = Bytes.putInt(shallowBuffer, pos, chunkD.getId()); // deep chunk index + pos = Bytes.putInt(shallowBuffer, pos, offset); // offset + pos = Bytes.putInt(shallowBuffer, pos, KeyValueUtil.length(kv2)); // length + offset += KeyValueUtil.length(kv2); + + KeyValueUtil.appendToByteArray(kv3, deepBuffer, offset); // write deep cell data + pos = Bytes.putInt(shallowBuffer, pos, chunkD.getId()); // deep chunk index + pos = Bytes.putInt(shallowBuffer, pos, offset); // offset + pos = Bytes.putInt(shallowBuffer, pos, KeyValueUtil.length(kv3)); // length + + return result; + } +} From 99ccfc0f61c45471631bcd215fd49c84cdc6d8f1 Mon Sep 17 00:00:00 2001 From: anastas Date: Wed, 30 Mar 2016 15:09:56 +0300 Subject: [PATCH 2/9] Connecting CellFlatMap to CompactingMemstore --- .../apache/hadoop/hbase/util/ClassSize.java | 11 + ...lockObjectArray.java => CellArrayMap.java} | 24 +- ...BlockSerialized.java => CellChunkMap.java} | 21 +- .../{CellBlock.java => CellFlatMap.java} | 58 ++-- .../regionserver/CompactingMemStore.java | 268 ++++++++++++------ .../regionserver/CompactionPipeline.java | 36 ++- .../hbase/regionserver/HeapMemStoreLAB.java | 21 +- .../hbase/regionserver/ImmutableSegment.java | 139 +++++++++ .../hbase/regionserver/MemStoreChunkPool.java | 5 +- .../MemStoreCompactorIterator.java | 163 +++++++++++ .../hbase/regionserver/MemStoreScanner.java | 62 ++-- .../hbase/regionserver/MutableSegment.java | 5 +- .../hadoop/hbase/regionserver/Segment.java | 62 +++- .../hbase/regionserver/SegmentFactory.java | 36 ++- .../regionserver/VersionedSegmentsList.java | 15 + .../hbase/regionserver/TestCellBlocksSet.java | 14 +- 16 files changed, 742 insertions(+), 198 deletions(-) rename hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/{CellBlockObjectArray.java => CellArrayMap.java} (58%) rename hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/{CellBlockSerialized.java => CellChunkMap.java} (79%) rename hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/{CellBlock.java => CellFlatMap.java} (89%) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorIterator.java diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java index fdd0faefa71c..1c1b2c190f98 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; + /** * Class for determining the "size" of a class, an attempt to calculate the * actual bytes that an object of this class will occupy in memory @@ -83,6 +84,12 @@ public class ClassSize { /** Overhead for ConcurrentSkipListMap Entry */ public static final int CONCURRENT_SKIPLISTMAP_ENTRY; + /** Overhead for CellArrayMap */ + public static final int CELL_ARRAY_MAP; + + /** Overhead for Cell Array Entry */ + public static final int CELL_ARRAY_ENTRY; + /** Overhead for ReentrantReadWriteLock */ public static final int REENTRANT_LOCK; @@ -174,10 +181,14 @@ public class ClassSize { // The size changes from jdk7 to jdk8, estimate the size rather than use a conditional CONCURRENT_SKIPLISTMAP = (int) estimateBase(ConcurrentSkipListMap.class, false); + CELL_ARRAY_MAP = align(OBJECT); + CONCURRENT_SKIPLISTMAP_ENTRY = align( align(OBJECT + (3 * REFERENCE)) + /* one node per entry */ align((OBJECT + (3 * REFERENCE))/2)); /* one index per two entries */ + CELL_ARRAY_ENTRY = align(2*REFERENCE + 2*Bytes.SIZEOF_INT); + REENTRANT_LOCK = align(OBJECT + (3 * REFERENCE)); ATOMIC_LONG = align(OBJECT + Bytes.SIZEOF_LONG); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockObjectArray.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java similarity index 58% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockObjectArray.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java index bebdd7aee5f7..8508abeb5a23 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockObjectArray.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java @@ -23,26 +23,26 @@ import org.apache.hadoop.hbase.Cell; /** - * CellBlockObjectArray is a simple array of Cells allocated using JVM. - * As all java arrays it is array of references pointing to Cell objects + * CellArrayMap is a simple array of Cells and can be allocated only using JVM. + * In contrast, CellChunkMap can be also allocated off-heap. + * As all java arrays CellArrayMap's array of references pointing to Cell objects. */ -public class CellBlockObjectArray extends CellBlock { +public class CellArrayMap extends CellFlatMap { - Cell[] block; + private final Cell[] block; - /* The Cells Array is created only when CellBlockObjectArray is created, all sub-CellBlocks use - * boundary indexes */ - public CellBlockObjectArray(Comparator comparator, Cell[] b, int min, int max, - boolean d) { + /* The Cells Array is created only when CellArrayMap is created, all sub-CellBlocks use + * boundary indexes. The given Cell array must be ordered. */ + public CellArrayMap(Comparator comparator, Cell[] b, int min, int max, boolean d) { super(comparator,min,max,d); this.block = b; } - /* To be used by base class only to create a sub-CellBlock */ + /* To be used by base class only to create a sub-CellFlatMap */ @Override - protected CellBlock createCellBlocks(Comparator comparator, - int min, int max, boolean d) { - return new CellBlockObjectArray(comparator,this.block,min,max,d); + protected CellFlatMap createCellFlatMap(Comparator comparator, int min, int max, + boolean d) { + return new CellArrayMap(comparator,this.block,min,max,d); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockSerialized.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java similarity index 79% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockSerialized.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java index 003cb952db66..2ed5a0dcd263 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockSerialized.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java @@ -27,7 +27,7 @@ /** - * CellBlockSerialized is a byte array holding all that is needed to access a Cell, which + * CellChunkMap is a byte array holding all that is needed to access a Cell, which * is actually saved on another deeper byte array. * Per Cell we have a reference to this deeper byte array B, offset in bytes in B (integer), * and length in bytes in B (integer). In order to save reference to byte array we use the Chunk's @@ -35,21 +35,22 @@ * * The B memory layout: * - * <----------------- first Cell ---------------------> <-------------- second Cell + * <----------------- first Cell ---------------------> <-------------- second Cell --- ... * ------------------------------------------------------------------------------------- ... * | integer = x bytes | integer = x bytes | integer = x bytes | integer = x bytes | * | reference to B | offset in B where | length of Cell's | reference to may be| ... * | holding Cell data | Cell's data starts| data in B | another byte array | * ------------------------------------------------------------------------------------- ... */ -public class CellBlockSerialized extends CellBlock { +public class CellChunkMap extends CellFlatMap { - private HeapMemStoreLAB.Chunk[] chunks; + private final HeapMemStoreLAB.Chunk[] chunks; private final HeapMemStoreLAB memStoreLAB; private int numOfCellsInsideChunk; - private static final int BYTES_IN_CELL = 3*(Integer.SIZE / Byte.SIZE); // each Cell requires 3 integers + public static final int BYTES_IN_CELL = 3*(Integer.SIZE / Byte.SIZE); // each Cell requires 3 integers - public CellBlockSerialized(Comparator comparator, HeapMemStoreLAB memStoreLAB, + /* The given Cell array on given Chunk array must be ordered. */ + public CellChunkMap(Comparator comparator, HeapMemStoreLAB memStoreLAB, HeapMemStoreLAB.Chunk[] chunks, int min, int max, int chunkSize, boolean d) { super(comparator,min,max, d); this.chunks = chunks; @@ -57,11 +58,11 @@ public CellBlockSerialized(Comparator comparator, HeapMemStoreLAB this.numOfCellsInsideChunk = chunkSize / BYTES_IN_CELL; } - /* To be used by base class only to create a sub-CellBlock */ + /* To be used by base class only to create a sub-CellFlatMap */ @Override - protected CellBlock createCellBlocks(Comparator comparator, - int min, int max, boolean d) { - return new CellBlockSerialized(comparator, this.memStoreLAB, this.chunks, min, max, + protected CellFlatMap createCellFlatMap(Comparator comparator, int min, int max, + boolean d) { + return new CellChunkMap(comparator, this.memStoreLAB, this.chunks, min, max, this.numOfCellsInsideChunk* BYTES_IN_CELL, d); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java similarity index 89% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlock.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java index 6403a5266f1f..18d30c7847cb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlock.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java @@ -31,13 +31,16 @@ /** - * CellBlock stores a constant number of elements and is immutable after creation stage. - * Due to being immutable the CellBlock can be implemented as array. - * The actual array is on- or off-heap and is implemented in concrete class derived from CellBlock. - * The CellBlock uses no synchronization primitives, it is assumed to be created by a + * CellFlatMap stores a constant number of elements and is immutable after creation stage. + * Due to being immutable the CellFlatMap can be implemented as array. + * The actual array can be on- or off-heap and is implemented in concrete class derived from CellFlatMap. + * The CellFlatMap uses no synchronization primitives, it is assumed to be created by a * single thread and then it can be read-only by multiple threads. + * + * The "flat" in the name, means that the memory layout of the Map is sequential array and thus + * requires less memory than ConcurrentSkipListMap. */ -public abstract class CellBlock implements ConcurrentNavigableMap { +public abstract class CellFlatMap implements ConcurrentNavigableMap { private final Comparator comparator; private int minCellIdx = 0; // the index of the minimal cell (for sub-sets) @@ -45,15 +48,15 @@ public abstract class CellBlock implements ConcurrentNavigableMap { private boolean descending = false; /* C-tor */ - public CellBlock(Comparator comparator, int min, int max, boolean d){ + public CellFlatMap(Comparator comparator, int min, int max, boolean d){ this.comparator = comparator; this.minCellIdx = min; this.maxCellIdx = max; this.descending = d; } - /* Used for abstract CellBlock creation, implemented by derived class */ - protected abstract CellBlock createCellBlocks(Comparator comparator, int min, + /* Used for abstract CellFlatMap creation, implemented by derived class */ + protected abstract CellFlatMap createCellFlatMap(Comparator comparator, int min, int max, boolean descending); /* Returns the i-th cell in the cell block */ @@ -126,24 +129,24 @@ public ConcurrentNavigableMap subMap( Cell fromKey, int fromIndex = (getValidIndex(fromKey, !fromInclusive)); if (fromIndex > toIndex) throw new IllegalArgumentException("inconsistent range"); - return createCellBlocks(comparator, fromIndex, toIndex, descending); + return createCellFlatMap(comparator, fromIndex, toIndex, descending); } @Override public ConcurrentNavigableMap headMap(Cell toKey, boolean inclusive) { int index = getValidIndex(toKey, inclusive); - return createCellBlocks(comparator, minCellIdx, index, descending); + return createCellFlatMap(comparator, minCellIdx, index, descending); } @Override public ConcurrentNavigableMap tailMap(Cell fromKey, boolean inclusive) { int index = (getValidIndex(fromKey, !inclusive)); - return createCellBlocks(comparator, index, maxCellIdx, descending); + return createCellFlatMap(comparator, index, maxCellIdx, descending); } @Override public ConcurrentNavigableMap descendingMap() { - return createCellBlocks(comparator, minCellIdx, maxCellIdx, true); + return createCellFlatMap(comparator, minCellIdx, maxCellIdx, true); } @Override @@ -151,12 +154,12 @@ public ConcurrentNavigableMap subMap(Cell k1, Cell k2) { return this.subMap(k1, true, k2, true); } - @Override + @Override public ConcurrentNavigableMap headMap(Cell k) { return this.headMap(k, true); } - @Override + @Override public ConcurrentNavigableMap tailMap(Cell k) { return this.tailMap(k, true); } @@ -251,42 +254,42 @@ public Cell get(Object o) { // -------------------------------- Entry's getters -------------------------------- // all interfaces returning Entries are unsupported because we are dealing only with the keys - @Override + @Override public Entry lowerEntry(Cell k) { throw new UnsupportedOperationException(); } - @Override + @Override public Entry higherEntry(Cell k) { throw new UnsupportedOperationException(); } - @Override + @Override public Entry ceilingEntry(Cell k) { throw new UnsupportedOperationException(); } - @Override + @Override public Entry floorEntry(Cell k) { throw new UnsupportedOperationException(); } - @Override + @Override public Entry firstEntry() { throw new UnsupportedOperationException(); } - @Override + @Override public Entry lastEntry() { throw new UnsupportedOperationException(); } - @Override + @Override public Entry pollFirstEntry() { throw new UnsupportedOperationException(); } - @Override + @Override public Entry pollLastEntry() { throw new UnsupportedOperationException(); } @@ -312,7 +315,7 @@ public Cell remove(Object o) { throw new UnsupportedOperationException(); } - @Override + @Override public boolean replace(Cell k, Cell v, Cell v1) { throw new UnsupportedOperationException(); } @@ -327,7 +330,7 @@ public Cell putIfAbsent(Cell k, Cell v) { throw new UnsupportedOperationException(); } - @Override + @Override public boolean remove(Object o, Object o1) { throw new UnsupportedOperationException(); } @@ -344,7 +347,7 @@ public NavigableSet navigableKeySet() { throw new UnsupportedOperationException(); } - @Override + @Override public NavigableSet descendingKeySet() { throw new UnsupportedOperationException(); } @@ -392,18 +395,17 @@ public void remove() { } } - // -------------------------------- Collection -------------------------------- private final class CellBlocksCollection implements Collection { @Override public int size() { - return CellBlock.this.size(); + return CellFlatMap.this.size(); } @Override public boolean isEmpty() { - return CellBlock.this.isEmpty(); + return CellFlatMap.this.isEmpty(); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index 709f5027db71..572f3eda50d4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -19,11 +19,7 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; @@ -32,13 +28,12 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.executor.EventHandler; import org.apache.hadoop.hbase.executor.EventType; import org.apache.hadoop.hbase.executor.ExecutorService; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -60,7 +55,12 @@ public class CompactingMemStore extends AbstractMemStore { public final static long DEEP_OVERHEAD_PER_PIPELINE_ITEM = ClassSize.align( ClassSize.TIMERANGE_TRACKER + ClassSize.CELL_SKIPLIST_SET + ClassSize.CONCURRENT_SKIPLISTMAP); + public final static long DEEP_OVERHEAD_PER_PIPELINE_FLAT_ARRAY_ITEM = ClassSize.align( + ClassSize.TIMERANGE_TRACKER + + ClassSize.CELL_SKIPLIST_SET + ClassSize.CELL_ARRAY_MAP); public final static double IN_MEMORY_FLUSH_THRESHOLD_FACTOR = 0.9; + public final static double COMPACTION_TRIGGER_REMAIN_FACTOR = 1; + public final static boolean CHECK_FOR_COMPACTION = false; private static final Log LOG = LogFactory.getLog(CompactingMemStore.class); private HStore store; @@ -73,6 +73,17 @@ public class CompactingMemStore extends AbstractMemStore { // A flag for tests only private final AtomicBoolean allowCompaction = new AtomicBoolean(true); + /** + * Types of CompactingMemStore + */ + public enum Type { + COMPACT_TO_SKIPLIST_MAP, + COMPACT_TO_ARRAY_MAP, + COMPACT_TO_CHUNK_MAP + } + + private Type type = Type.COMPACT_TO_SKIPLIST_MAP; + public CompactingMemStore(Configuration conf, CellComparator c, HStore store, RegionServicesForStores regionServices) throws IOException { super(conf, c); @@ -83,6 +94,17 @@ public CompactingMemStore(Configuration conf, CellComparator c, initFlushSizeLowerBound(conf); } + public CompactingMemStore(Configuration conf, CellComparator c, + HStore store, RegionServicesForStores regionServices, Type type) throws IOException { + super(conf, c); + this.store = store; + this.regionServices = regionServices; + this.pipeline = new CompactionPipeline(getRegionServices()); + this.compactor = new MemStoreCompactor(); + this.type = type; + initFlushSizeLowerBound(conf); + } + private void initFlushSizeLowerBound(Configuration conf) { String flushedSizeLowerBoundString = getRegionServices().getTableDesc().getValue(FlushLargeStoresPolicy @@ -364,9 +386,10 @@ public void process() throws IOException { } } - /** - * The ongoing MemStore Compaction manager, dispatches a solo running compaction - * and interrupts the compaction if requested. + /** ---------------------------------------------------------------------- + * The ongoing MemStore Compaction manager, dispatches a solo running compaction and interrupts + * the compaction if requested. Prior to compaction the MemStoreCompactor evaluates + * the compacting ratio and aborts the compaction if it is not worthy. * The MemStoreScanner is used to traverse the compaction pipeline. The MemStoreScanner * is included in internal store scanner, where all compaction logic is implemented. * Threads safety: It is assumed that the compaction pipeline is immutable, @@ -374,47 +397,44 @@ public void process() throws IOException { */ private class MemStoreCompactor { - private MemStoreScanner scanner; // scanner for pipeline only - // scanner on top of MemStoreScanner that uses ScanQueryMatcher - private StoreScanner compactingScanner; - - // smallest read point for any ongoing MemStore scan - private long smallestReadPoint; + // list of Scanners of segments in the pipeline, when compaction starts + List scanners = new ArrayList(); // a static version of the segment list from the pipeline private VersionedSegmentsList versionedList; + + // a flag raised when compaction is requested to stop private final AtomicBoolean isInterrupted = new AtomicBoolean(false); - /** - * ---------------------------------------------------------------------- + // the limit to the size of the groups to be later returned by compactingScanner + private final int compactionKVMax = getConfiguration().getInt( + HConstants.COMPACTION_KV_MAX, + HConstants.COMPACTION_KV_MAX_DEFAULT); + + /** ---------------------------------------------------------------------- * The request to dispatch the compaction asynchronous task. * The method returns true if compaction was successfully dispatched, or false if there * * is already an ongoing compaction (or pipeline is empty). */ public boolean startCompact() throws IOException { + + //org.junit.Assert.assertTrue("\n\n<<< Starting compaction\n", false); if (pipeline.isEmpty()) return false; // no compaction on empty pipeline - List scanners = new ArrayList(); - // get the list of segments from the pipeline + // get the list of segments from the pipeline, the list is marked with specific version versionedList = pipeline.getVersionedList(); - // the list is marked with specific version + //org.junit.Assert.assertTrue("\n\n<<< Starting compaction, got versioned list\n", false); // create the list of scanners with maximally possible read point, meaning that // all KVs are going to be returned by the pipeline traversing for (Segment segment : versionedList.getStoreSegments()) { scanners.add(segment.getSegmentScanner(Long.MAX_VALUE)); } - scanner = - new MemStoreScanner(CompactingMemStore.this, scanners, Long.MAX_VALUE, - MemStoreScanner.Type.COMPACT_FORWARD); - - smallestReadPoint = store.getSmallestReadPoint(); - compactingScanner = createScanner(store); - - LOG.info("Starting the MemStore in-memory compaction for store " + - store.getColumnFamilyName()); + LOG.info( + "Starting the MemStore in-memory compaction for store " + store.getColumnFamilyName()); + //org.junit.Assert.assertTrue("\n\n<<< Starting actual compaction\n", false); doCompact(); return true; } @@ -435,10 +455,6 @@ public void stopCompact() { */ private void releaseResources() { isInterrupted.set(false); - scanner.close(); - scanner = null; - compactingScanner.close(); - compactingScanner = null; versionedList = null; } @@ -448,27 +464,65 @@ private void releaseResources() { * There is at most one thread per memstore instance. */ private void doCompact() { + int cellsAfterComp = versionedList.getNumOfCells(); + ImmutableSegment result = null; + + //org.junit.Assert.assertTrue("\n\n<<< Starting doCompact() 1\n", false); + + try { // the compaction processing + + if (CHECK_FOR_COMPACTION) { + //org.junit.Assert.assertTrue("\n\n<<< Starting doCompact() 2\n", false); + // Phase I: estimate the compaction expedience - CHECK COMPACTION + cellsAfterComp = countCellsForCompaction(); + // org.junit.Assert.assertTrue("\n\n<<< In compaction, the number of cells after " + // + "compaction should be: " + cellsAfterComp + " \n", false); + + if (!isInterrupted.get() && (cellsAfterComp + > COMPACTION_TRIGGER_REMAIN_FACTOR * versionedList.getNumOfCells())) { + // too much cells "survive" the possible compaction we do not want to compact! + LOG.debug("Stopping the unworthy MemStore in-memory compaction for store " + + getFamilyName()); + // Looking for Segment in pipeline snapshot with SkipList index, to make it flat + ImmutableSegment segment = versionedList.getSkipListSegment(); + if (segment == null) { + Thread.currentThread().interrupt(); + return; + } + // We are going to flatten the given segment with SkipList index + flattenSegment(segment, cellsAfterComp); + return; + } + } +// org.junit.Assert.assertTrue("\n\n<<< In compaction, decided to compact! " +// + "The number of cells after " + "compaction should be: " + cellsAfterComp + " \n", false); - ImmutableSegment result = SegmentFactory.instance() // create the scanner - .createImmutableSegment(getConfiguration(), getComparator(), - CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_ITEM); + // Phase II: create the compacted ImmutableSegment - START COMPACTION + if (!isInterrupted.get()) { + switch (type) { + case COMPACT_TO_SKIPLIST_MAP: result = compactToNotFlatSegment(); + case COMPACT_TO_ARRAY_MAP: result = compactToFlatSegment(cellsAfterComp, true); + case COMPACT_TO_CHUNK_MAP: result = compactToFlatSegment(cellsAfterComp, false); + default: assert false; // sanity check + } + } - // the compaction processing - try { - // Phase I: create the compacted MutableCellSetSegment - compactSegments(result); +// org.junit.Assert.assertTrue("\n\n<<< After compaction! " +// + "The number of cells after " + "compaction should be: " + cellsAfterComp +// + " \n", false); - // Phase II: swap the old compaction pipeline + // Phase III: swap the old compaction pipeline - END COMPACTION if (!isInterrupted.get()) { pipeline.swap(versionedList, result); // update the wal so it can be truncated and not get too long updateLowestUnflushedSequenceIdInWal(true); // only if greater } } catch (Exception e) { + org.junit.Assert.assertTrue("\n\n<<< GOT EXCEPTION IN COMPACTION!!! " + e + "\n", false); LOG.debug("Interrupting the MemStore in-memory compaction for store " + getFamilyName()); Thread.currentThread().interrupt(); - return; } finally { + //org.junit.Assert.assertTrue("\n\n<<< BEFORE RELEASE RESOURCE!!! " + "\n", false); releaseResources(); inMemoryFlushInProgress.set(false); } @@ -476,55 +530,107 @@ private void doCompact() { } /** - * Creates the scanner for compacting the pipeline. - * - * @return the scanner + * Creates a new Segment with CellFlatMap from the given single old + * Segment based on ConcurrentSkipListMap */ - private StoreScanner createScanner(Store store) throws IOException { - - Scan scan = new Scan(); - scan.setMaxVersions(); //Get all available versions + private void flattenSegment(ImmutableSegment segment, int numOfCells) throws IOException { - StoreScanner internalScanner = - new StoreScanner(store, store.getScanInfo(), scan, Collections.singletonList(scanner), - ScanType.COMPACT_RETAIN_DELETES, smallestReadPoint, HConstants.OLDEST_TIMESTAMP); + LOG.debug("Flattening Segment " + segment); + // Update the pipeline scanner (list of scanners) to scan only Skip List based Segment + List scanners = new ArrayList(); + scanners.add(segment.getSegmentScanner(Long.MAX_VALUE)); + ImmutableSegment newSegment = createFlatFromSegmentWithSkipList(numOfCells, segment); + pipeline.replace(versionedList, segment, newSegment); - return internalScanner; + return; } /** + * COMPACT TO ARRAY MAP * Updates the given single Segment using the internal store scanner, * who in turn uses ScanQueryMatcher */ - private void compactSegments(Segment result) throws IOException { - - List kvs = new ArrayList(); - // get the limit to the size of the groups to be returned by compactingScanner - int compactionKVMax = getConfiguration().getInt( - HConstants.COMPACTION_KV_MAX, - HConstants.COMPACTION_KV_MAX_DEFAULT); - - ScannerContext scannerContext = - ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build(); - - boolean hasMore; - do { - hasMore = compactingScanner.next(kvs, scannerContext); - if (!kvs.isEmpty()) { - for (Cell c : kvs) { - // The scanner is doing all the elimination logic - // now we just copy it to the new segment - KeyValue kv = KeyValueUtil.ensureKeyValue(c); - Cell newKV = result.maybeCloneWithAllocator(kv); - result.internalAdd(newKV); + private ImmutableSegment compactToFlatSegment(int numOfCells, boolean array) + throws IOException { - } - kvs.clear(); - } - } while (hasMore && (!isInterrupted.get())); + MemStoreCompactorIterator iterator = + new MemStoreCompactorIterator( + 2, versionedList.getStoreSegments(), getComparator(), compactionKVMax, store); + +// org.junit.Assert.assertTrue("\n\n<<< After creating the scanner in compaction\n", false); + ImmutableSegment result = SegmentFactory.instance() + .createImmutableSegment(getConfiguration(), getComparator(), numOfCells, iterator, array); + + iterator.remove(); + return result; } - } + /** + * COMPACT TO SKIPLIST MAP + */ + private ImmutableSegment compactToNotFlatSegment() throws IOException { + ImmutableSegment result = SegmentFactory.instance() // create the scanner + .createImmutableSegment(getConfiguration(), getComparator(), + CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_ITEM); + + //org.junit.Assert.assertTrue("\n\n<<< Starting counting cells for comaction 1\n", false); + MemStoreCompactorIterator iterator = + new MemStoreCompactorIterator( + 3,versionedList.getStoreSegments(),getComparator(),compactionKVMax, store); + + //org.junit.Assert.assertTrue("\n\n<<< Starting counting cells for comaction 2\n", false); + while (iterator.hasNext()) { + Cell c = iterator.next(); + // The scanner is doing all the elimination logic + // now we just copy it to the new segment + KeyValue kv = KeyValueUtil.ensureKeyValue(c); + Cell newKV = result.maybeCloneWithAllocator(kv); + result.internalAdd(newKV); + } + iterator.remove(); + // org.junit.Assert.assertTrue("\n\n<<< Finished counting " + // + cnt + " cells for comaction\n", false); + return result; + } + + /** + * FLATTENING + * Create CellSet for flat representation + * The MSLAB Chunks with the real data remain the same + */ + private ImmutableSegment createFlatFromSegmentWithSkipList(int numOfCells, + ImmutableSegment segment) throws IOException { + + LinkedList segmentList = new LinkedList(); + segmentList.add(segment); + + MemStoreCompactorIterator iterator = + new MemStoreCompactorIterator(4,segmentList,getComparator(),compactionKVMax, store); + + ImmutableSegment result = SegmentFactory.instance() + .createImmutableSegment(numOfCells, segment, iterator); + iterator.remove(); + return result; + } + + private int countCellsForCompaction() throws IOException { + + //org.junit.Assert.assertTrue("\n\n<<< Starting counting cells for comaction 1\n", false); + MemStoreCompactorIterator iterator = + new MemStoreCompactorIterator( + 3,versionedList.getStoreSegments(),getComparator(),compactionKVMax, store); + int cnt = 0; + + //org.junit.Assert.assertTrue("\n\n<<< Starting counting cells for comaction 2\n", false); + while (iterator.next() != null) { + cnt++; + } + iterator.remove(); +// org.junit.Assert.assertTrue("\n\n<<< Finished counting " +// + cnt + " cells for comaction\n", false); + return cnt; + } + } // end of the MemStoreCompactor Class //---------------------------------------------------------------------- //methods for tests diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java index 165406528f64..f75aeba1c813 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java @@ -112,6 +112,32 @@ public boolean swap(VersionedSegmentsList versionedList, ImmutableSegment segmen return true; } + /** + * Replaces a single segment in the pipeline with the new flattened segment. + * Replacing only if there were no changes in the pipeline. This is achieved + * by requiring the version (from versionedList) being the same. + * @param versionedList tail of the pipeline that was taken for compaction and holds the version + * @param newSegment new compacted segment + * @return true iff replaced with new compacted segment + */ + public boolean replace(VersionedSegmentsList versionedList, + ImmutableSegment oldSegment, ImmutableSegment newSegment) { + + if(versionedList.getVersion() != version) { + return false; + } + + synchronized (pipeline){ + if(versionedList.getVersion() != version) { + return false; + } + LOG.info("Replacing one pipeline segment with flattened segment."); + replaceSegment(oldSegment,newSegment); + } + // do not update the global memstore size counter, because all the cells remain in place + return true; + } + public boolean isEmpty() { return pipeline.isEmpty(); } @@ -149,6 +175,15 @@ private void swapSuffix(LinkedList suffix, ImmutableSegment se pipeline.addLast(segment); } + /* Replacing one representation of the segment with another. Literally removing a segment and + adding a new one (with same keys). The order of the segments is not kept */ + private void replaceSegment(ImmutableSegment oldSegment, ImmutableSegment newSegment) { + version++; + // don't call oldSegment.close() because its MSLAB is transferred to the new segment + pipeline.remove(oldSegment); + pipeline.add(newSegment); + } + private ImmutableSegment removeLast() { version++; return pipeline.removeLast(); @@ -165,7 +200,6 @@ private boolean validateSuffixList(LinkedList suffix) { // empty suffix is always valid return true; } - Iterator pipelineBackwardIterator = pipeline.descendingIterator(); Iterator suffixBackwardIterator = suffix.descendingIterator(); ImmutableSegment suffixCurrent; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java index 228616bade7e..05b5febf250b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java @@ -193,12 +193,13 @@ private Chunk getOrMakeChunk() { // against other allocators to CAS in an uninitialized chunk // (which is cheap to allocate) - //c = (chunkPool != null) ? chunkPool.getChunk() : new Chunk(chunkSize, 5); //14921 + //c = (chunkPool != null) ? chunkPool.getChunk() : new Chunk(chunkSize, 5); //HBASE-14921 if(chunkPool != null) { c = chunkPool.getChunk(); } else { - c = new Chunk(chunkSize, 5); + // HBASE-14921 555 is here till it is decided whether ChunkPool is always on + c = new Chunk(chunkSize, 555); c.init(); } @@ -217,19 +218,27 @@ private Chunk getOrMakeChunk() { } } - /** 14921 + /** HBASE-14921 * Given a chunk ID return reference to the relevant chunk * @return a chunk */ - Chunk translateIdToChunk(int id) { + public Chunk translateIdToChunk(int id) { return chunkPool.translateIdToChunk(id); } - /** 14921 + /** HBASE-14921 + * Give the ID of the Chunk from where last allocation took the bytes + * @return a chunk + */ + public int getCurrentChunkId() { + return curChunk.get().getId(); + } + + /** HBASE-14921 * Use instead of allocateBytes() when new full chunk is needed * @return a chunk */ - Chunk allocateChunk() { + public Chunk allocateChunk() { Chunk c = chunkPool.getChunk(); this.chunkQueue.add(c); return c; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java index 077c27095877..4d2c67cc2d5c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java @@ -18,7 +18,14 @@ */ package org.apache.hadoop.hbase.regionserver; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.ByteRange; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CollectionBackedScanner; /** @@ -30,8 +37,114 @@ @InterfaceAudience.Private public class ImmutableSegment extends Segment { + private boolean isFlat; // whether it is based on CellFlatMap or ConcurrentSkipListMap + protected ImmutableSegment(Segment segment) { super(segment); + isFlat = false; + } + + // flattening + protected ImmutableSegment(ImmutableSegment oldSegment, MemStoreCompactorIterator iterator, + MemStoreLAB memStoreLAB, int numOfCells, long constantCellSizeOverhead) { + + super(null,oldSegment.getComparator(),memStoreLAB, + CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_FLAT_ARRAY_ITEM, constantCellSizeOverhead); + + // build the CellSet + CellSet cs = this.createArrayBasedCellSet(numOfCells, iterator, false); + + // update the CellSet of the new Segment + this.setCellSet(cs); + isFlat = true; + } + + // compaction to CellArrayMap + protected ImmutableSegment( + final Configuration conf, CellComparator comparator, MemStoreCompactorIterator iterator, + MemStoreLAB memStoreLAB, int numOfCells, long constantCellSizeOverhead, boolean array) { + + super(null, comparator, memStoreLAB, + CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_FLAT_ARRAY_ITEM, constantCellSizeOverhead); + +// org.junit.Assert.assertTrue("\n\n<<< Immutable Segment Constructor BEFORE compaction\n", false); + + // build the CellSet + CellSet cs = null; + if (array) { + cs = this.createArrayBasedCellSet(numOfCells, iterator, true); + } else { + cs = this.createChunkBasedCellSet(numOfCells, iterator, conf); + } + + // update the CellSet of the new Segment + this.setCellSet(cs); + isFlat = true; + } + + private CellSet createArrayBasedCellSet(int numOfCells, MemStoreCompactorIterator iterator, + boolean allocateFromMSLAB) { + // build the Cell Array + Cell[] cells = new Cell[numOfCells]; + int i = 0; + while (iterator.hasNext()) { + Cell c = iterator.next(); + if (allocateFromMSLAB) { + // The scanner behind the iterator is doing all the elimination logic + // now we just copy it to the new segment (also MSLAB copy) + KeyValue kv = KeyValueUtil.ensureKeyValue(c); + Cell newKV = maybeCloneWithAllocator(kv); + cells[i++] = newKV; + // flattening = !allocateFromMSLAB (false), counting both Heap and MetaData size + updateMetaInfo(c,true,!allocateFromMSLAB); + } else { + cells[i++] = c; + // flattening = !allocateFromMSLAB (true), counting only MetaData size + updateMetaInfo(c,true,!allocateFromMSLAB); + } + + } + // build the immutable CellSet + CellArrayMap + cam = new CellArrayMap(getComparator(),cells,0,numOfCells,false); + return new CellSet(cam); + } + + private CellSet createChunkBasedCellSet( + int numOfCells, MemStoreCompactorIterator iterator, final Configuration conf) { + + int chunkSize = conf.getInt(HeapMemStoreLAB.CHUNK_SIZE_KEY, HeapMemStoreLAB.CHUNK_SIZE_DEFAULT); + int numOfCellsInsideChunk = chunkSize / CellChunkMap.BYTES_IN_CELL; + int numberOfChunks = chunkSize/numOfCellsInsideChunk; + HeapMemStoreLAB ms = (HeapMemStoreLAB)getMemStoreLAB(); + + // all Chunks must be allocated from current MSLAB + HeapMemStoreLAB.Chunk[] chunks = new HeapMemStoreLAB.Chunk[numberOfChunks]; + int currentChunkIdx = 0; + chunks[currentChunkIdx] = ms.allocateChunk(); + int offsetInCurentChunk = 0; + + while (iterator.hasNext()) { + Cell c = iterator.next(); + + if (offsetInCurentChunk + CellChunkMap.BYTES_IN_CELL > chunkSize) { + // we do not consider cells bigger than chunks + // continue to the next chunk + currentChunkIdx++; + chunks[currentChunkIdx] = ms.allocateChunk(); + offsetInCurentChunk = 0; + } + + // The scanner behind the iterator is doing all the elimination logic + // now we just copy it to the new segment (also MSLAB copy) + KeyValue kv = KeyValueUtil.ensureKeyValue(c); + offsetInCurentChunk = + cloneAndReference(kv, chunks[currentChunkIdx].getData(), offsetInCurentChunk); + } + + CellChunkMap ccm = new CellChunkMap(getComparator(),(HeapMemStoreLAB)getMemStoreLAB(), + chunks,0,numOfCells,chunkSize,false); + return new CellSet(ccm); } /** @@ -43,4 +156,30 @@ public KeyValueScanner getKeyValueScanner() { return new CollectionBackedScanner(getCellSet(), getComparator()); } + public boolean isFlat() { + return isFlat; + } + + /** + * If the segment has a memory allocator the cell is being cloned to this space, and returned; + * otherwise the given cell is returned + * @return either the given cell or its clone + */ + private int cloneAndReference(Cell cell, byte[] referencesByteArray, int offsetForReference) { + HeapMemStoreLAB ms = (HeapMemStoreLAB)getMemStoreLAB(); + int len = KeyValueUtil.length(cell); + int offset = offsetForReference; + // we assume Cell length is not bigger than Chunk + + // allocate + ByteRange alloc = ms.allocateBytes(len); + int chunkId = ms.getCurrentChunkId(); + KeyValueUtil.appendToByteArray(cell, alloc.getBytes(), alloc.getOffset()); + + // write the reference + offset = Bytes.putInt(referencesByteArray, offset, chunkId); // write chunk id + offset = Bytes.putInt(referencesByteArray, offset, alloc.getOffset()); // offset + offset = Bytes.putInt(referencesByteArray, offset, len); // length + return offset; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java index bc85bc0e95ee..cf2231e2f675 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java @@ -18,6 +18,7 @@ */ package org.apache.hadoop.hbase.regionserver; +import com.google.common.annotations.VisibleForTesting; import java.lang.management.ManagementFactory; //import java.util.concurrent.*; import java.util.concurrent.BlockingQueue; @@ -158,9 +159,7 @@ int getPoolSize() { return this.reclaimedChunks.size(); } - /* - * Only used in testing - */ + @VisibleForTesting void clearChunks() { this.reclaimedChunks.clear(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorIterator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorIterator.java new file mode 100644 index 000000000000..6c813a0dd605 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorIterator.java @@ -0,0 +1,163 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.Scan; + +import java.io.IOException; +import java.util.*; + +/** + * The MemStoreCompactorIterator is designed to perform one iteration over given list of segments + * For another iteration new instance of MemStoreCompactorIterator needs to be created + * The iterator is not thread-safe and must have only one instance in each period of time + */ +@InterfaceAudience.Private +public class MemStoreCompactorIterator implements Iterator { + + private List kvs = new ArrayList(); + private int c = 0; + // scanner for full or partial pipeline (heap of segment scanners) + private KeyValueScanner scanner; + // scanner on top of pipeline scanner that uses ScanQueryMatcher + private StoreScanner compactingScanner; + // get the limit to the size of the groups to be returned by compactingScanner + private final int compactionKVMax; + + private final ScannerContext scannerContext; + + private boolean hasMore; + private Iterator kvsIterator; + + // C-tor + public MemStoreCompactorIterator(int initiator, LinkedList segments, + CellComparator comparator, int compactionKVMax, HStore store) throws IOException { + this.compactionKVMax = compactionKVMax; + this.scannerContext = + ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build(); + //org.junit.Assert.assertTrue("\n\n<<< Initiating compactor iterator\n", false); + + // list of Scanners of segments in the pipeline, when compaction starts + List scanners = new ArrayList(); + + // create the list of scanners with maximally possible read point, meaning that + // all KVs are going to be returned by the pipeline traversing + for (Segment segment : segments) { + scanners.add(segment.getSegmentScanner(Long.MAX_VALUE)); + } + + scanner = + new MemStoreScanner(comparator, scanners, MemStoreScanner.Type.COMPACT_FORWARD); + + // reinitialize the compacting scanner for each instance of iterator + compactingScanner = createScanner(store, scanner, initiator); + + hasMore = compactingScanner.next(kvs, scannerContext); + if (!kvs.isEmpty()) { + kvsIterator = kvs.iterator(); + } + +// if (initiator == 2) +// org.junit.Assert.assertTrue("\n\n<<< Iterator constructing. " +// + "Store: " + store + ", Scanner: " + scanner + ", hasMore: " + hasMore + "\n", false); + } + + @Override + public boolean hasNext() { + if (!kvsIterator.hasNext()) { + if (!refillKVS()) return false; + } + return hasMore; + } + + @Override + public Cell next() { + c++; + //if (c>1) org.junit.Assert.assertTrue("\n\n<<< " + c + " iterations of iterator\n",false); + if (!kvsIterator.hasNext()) { + // org.junit.Assert.assertTrue("\n\n<<< Got to empty kvs for the first time with " + c + // + " iterations\n", false); + if (!refillKVS()) return null; + } + + return (!hasMore) ? null : kvsIterator.next(); + } + + @Override + public void remove() { + compactingScanner.close(); + compactingScanner = null; + scanner.close(); + scanner = null; + } + + /** + * Creates the scanner for compacting the pipeline. + * + * @return the scanner + */ + private StoreScanner createScanner(Store store, KeyValueScanner scanner, int init) + throws IOException { + + Scan scan = new Scan(); + scan.setMaxVersions(); //Get all available versions + +// if (init == 2) +// org.junit.Assert.assertTrue("\n\n<<< Before creating the internal scanner " + "\n", false); + + StoreScanner internalScanner = + new StoreScanner(store, store.getScanInfo(), scan, Collections.singletonList(scanner), + ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), + HConstants.OLDEST_TIMESTAMP); + + return internalScanner; + } + + private boolean refillKVS() { + kvs.clear(); // clear previous KVS, first initiated in the constructor + + if (!hasMore) { // if there is nothing expected next in compactingScanner + org.junit.Assert.assertTrue("\n\n<<< Got to empty kvs for the first time with " + c + + " iterations and no more\n", false); + return false; + } + + try { // try to get next KVS + hasMore = compactingScanner.next(kvs, scannerContext); + } catch (IOException ie) { + org.junit.Assert.assertTrue("\n\n<<< GOT EXCEPTION IN ITERATOR !!!\n",false); + throw new IllegalStateException(ie); + } + + if (!kvs.isEmpty() ) { // is the new KVS empty ? + kvsIterator = kvs.iterator(); + return true; + } else { + // org.junit.Assert.assertTrue("\n\n<<< Got to empty kvs for the first time with " + c + // + " iterations and empty set\n", false); + return false; + } + } +} + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreScanner.java index dfcec25e1b10..61f00c079d9f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreScanner.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Scan; @@ -55,12 +56,11 @@ static public enum Type { // or according to the first usage private Type type = Type.UNDEFINED; - private long readPoint; // remember the initial version of the scanners list List scanners; - // pointer back to the relevant MemStore - // is needed for shouldSeek() method - private AbstractMemStore backwardReferenceToMemStore; + + private final CellComparator comparator; + /** * Constructor. @@ -68,44 +68,40 @@ static public enum Type { * After constructor only one heap is going to be initialized for entire lifespan * of the MemStoreScanner. A specific scanner can only be one directional! * - * @param ms Pointer back to the MemStore - * @param readPoint Read point below which we can safely remove duplicate KVs - * @param type The scan type COMPACT_FORWARD should be used for compaction + * @param comparator Cell Comparator + * @param scanners List of scanners, from which the heap will be built + * @param type The scan type COMPACT_FORWARD should be used for compaction */ - public MemStoreScanner(AbstractMemStore ms, long readPoint, Type type) throws IOException { - this(ms, ms.getListOfScanners(readPoint), readPoint, type); - } - - /* Constructor used only when the scan usage is unknown - and need to be defined according to the first move */ - public MemStoreScanner(AbstractMemStore ms, long readPt) throws IOException { - this(ms, readPt, Type.UNDEFINED); - } - - public MemStoreScanner(AbstractMemStore ms, List scanners, long readPoint, - Type type) throws IOException { + public MemStoreScanner(CellComparator comparator, List scanners, Type type) + throws IOException { super(); - this.readPoint = readPoint; this.type = type; switch (type) { - case UNDEFINED: - case USER_SCAN_FORWARD: - case COMPACT_FORWARD: - this.forwardHeap = new KeyValueHeap(scanners, ms.getComparator()); - break; - case USER_SCAN_BACKWARD: - this.backwardHeap = new ReversedKeyValueHeap(scanners, ms.getComparator()); - break; - default: - throw new IllegalArgumentException("Unknown scanner type in MemStoreScanner"); + case UNDEFINED: + case USER_SCAN_FORWARD: + case COMPACT_FORWARD: + this.forwardHeap = new KeyValueHeap(scanners, comparator); + break; + case USER_SCAN_BACKWARD: + this.backwardHeap = new ReversedKeyValueHeap(scanners, comparator); + break; + default: + throw new IllegalArgumentException("Unknown scanner type in MemStoreScanner"); } - this.backwardReferenceToMemStore = ms; + this.comparator = comparator; this.scanners = scanners; if (Trace.isTracing() && Trace.currentSpan() != null) { Trace.currentSpan().addTimelineAnnotation("Creating MemStoreScanner"); } } + /* Constructor used only when the scan usage is unknown + and need to be defined according to the first move */ + public MemStoreScanner(CellComparator comparator, List scanners) + throws IOException { + this(comparator, scanners, Type.UNDEFINED); + } + /** * Returns the cell from the top-most scanner without advancing the iterator. * The backward traversal is assumed, only if specified explicitly @@ -292,7 +288,7 @@ private boolean restartBackwardHeap(Cell cell) throws IOException { res |= scan.seekToPreviousRow(cell); } this.backwardHeap = - new ReversedKeyValueHeap(scanners, backwardReferenceToMemStore.getComparator()); + new ReversedKeyValueHeap(scanners, comparator); return res; } @@ -322,7 +318,7 @@ private boolean initBackwardHeapIfNeeded(Cell cell, boolean toLast) throws IOExc } } this.backwardHeap = - new ReversedKeyValueHeap(scanners, backwardReferenceToMemStore.getComparator()); + new ReversedKeyValueHeap(scanners, comparator); type = Type.USER_SCAN_BACKWARD; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java index aef70e710c75..73766261f0db 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.ClassSize; /** * A mutable segment in memstore, specifically the active segment. @@ -30,7 +31,7 @@ public class MutableSegment extends Segment { protected MutableSegment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, long size) { - super(cellSet, comparator, memStoreLAB, size); + super(cellSet, comparator, memStoreLAB, size, ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); } /** @@ -48,7 +49,7 @@ public long add(Cell cell) { public long rollback(Cell cell) { Cell found = getCellSet().get(cell); if (found != null && found.getSequenceId() == cell.getSequenceId()) { - long sz = AbstractMemStore.heapSizeChange(cell, true); + long sz = AbstractMemStore.heapSizeChange(cell, true, false); getCellSet().remove(cell); incSize(-sz); return sz; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java index 34d0a51a27bb..1e0c5a92a5cd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java @@ -23,13 +23,11 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.ByteRange; +import org.apache.hadoop.hbase.util.ClassSize; /** * This is an abstraction of a segment maintained in a memstore, e.g., the active @@ -47,10 +45,13 @@ public abstract class Segment { private volatile MemStoreLAB memStoreLAB; private final AtomicLong size; private final TimeRangeTracker timeRangeTracker; + private long constantCellMetaDataSize; protected volatile boolean tagsPresent; + protected boolean hasFlatIndex; - protected Segment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, long - size) { + protected Segment( + CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, long size, + long constantCellSize) { this.cellSet = cellSet; this.comparator = comparator; this.minSequenceId = Long.MAX_VALUE; @@ -58,6 +59,8 @@ protected Segment(CellSet cellSet, CellComparator comparator, MemStoreLAB memSto this.size = new AtomicLong(size); this.timeRangeTracker = new TimeRangeTracker(); this.tagsPresent = false; + this.hasFlatIndex = true; + this.constantCellMetaDataSize = constantCellSize; } protected Segment(Segment segment) { @@ -68,6 +71,7 @@ protected Segment(Segment segment) { this.size = new AtomicLong(segment.getSize()); this.timeRangeTracker = segment.getTimeRangeTracker(); this.tagsPresent = segment.isTagsPresent(); + this.constantCellMetaDataSize = segment.getConstantCellMetaDataSize(); } /** @@ -156,6 +160,10 @@ public boolean isTagsPresent() { return tagsPresent; } + public boolean hasFlatIndex() { + return hasFlatIndex; + } + public void incScannerCount() { if(getMemStoreLAB() != null) { getMemStoreLAB().incScannerCount(); @@ -178,6 +186,17 @@ public Segment setSize(long size) { return this; } + /** + * Setting the CellSet of the segment - used only for flat immutable segment for setting + * immutable CellSet after its creation in immutable segment constructor + * @return this object + */ + + public Segment setCellSet(CellSet cellSet) { + this.cellSet = cellSet; + return this; + } + /** * Returns the heap size of the segment * @return the heap size of the segment @@ -240,22 +259,23 @@ protected CellComparator getComparator() { protected long internalAdd(Cell cell) { boolean succ = getCellSet().add(cell); - long s = AbstractMemStore.heapSizeChange(cell, succ); - updateMetaInfo(cell, s); + long s = updateMetaInfo(cell, succ, false); return s; } - protected void updateMetaInfo(Cell toAdd, long s) { - getTimeRangeTracker().includeTimestamp(toAdd); + protected long updateMetaInfo(Cell cellToAdd, boolean succ, boolean flattenning) { + long s = flattenning ? constantCellMetaDataSize : heapSizeChange(cellToAdd, succ); + getTimeRangeTracker().includeTimestamp( cellToAdd); size.addAndGet(s); - minSequenceId = Math.min(minSequenceId, toAdd.getSequenceId()); + minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId()); // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call. // When we use ACL CP or Visibility CP which deals with Tags during // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not // parse the byte[] to identify the tags length. - if(toAdd.getTagsLength() > 0) { + if( cellToAdd.getTagsLength() > 0) { tagsPresent = true; } + return s; } /** @@ -267,7 +287,7 @@ protected SortedSet tailSet(Cell firstCell) { return getCellSet().tailSet(firstCell); } - private MemStoreLAB getMemStoreLAB() { + protected MemStoreLAB getMemStoreLAB() { return memStoreLAB; } @@ -291,4 +311,20 @@ public String toString() { return res; } + /* + * Calculate how the MemStore size has changed. Includes overhead of the + * backing Map. + * @param cell + * @param notPresent True if the cell was NOT present in the set. + * @return change in size + */ + protected long heapSizeChange(final Cell cell, final boolean notPresent){ + return + notPresent ? + ClassSize.align(constantCellMetaDataSize + CellUtil.estimatedHeapSizeOf(cell)) : 0; + } + + public long getConstantCellMetaDataSize() { + return this.constantCellMetaDataSize; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java index 7ac80ae27749..7b11c08b89e1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java @@ -19,10 +19,15 @@ package org.apache.hadoop.hbase.regionserver; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.ReflectionUtils; +import java.io.IOException; +import java.util.List; + /** * A singleton store segment factory. * Generate concrete store segments. @@ -40,6 +45,7 @@ public static SegmentFactory instance() { return instance; } + // create skip-list-based immutable segment public ImmutableSegment createImmutableSegment(final Configuration conf, final CellComparator comparator, long size) { MemStoreLAB memStoreLAB = getMemStoreLAB(conf); @@ -47,21 +53,47 @@ public ImmutableSegment createImmutableSegment(final Configuration conf, return createImmutableSegment(segment); } - public ImmutableSegment createImmutableSegment(CellComparator comparator, - long size) { + // usually used to create empty immutable segment + public ImmutableSegment createImmutableSegment(CellComparator comparator, long size) { MutableSegment segment = generateMutableSegment(null, comparator, null, size); return createImmutableSegment(segment); } + // create immutable segment from mutable public ImmutableSegment createImmutableSegment(MutableSegment segment) { return new ImmutableSegment(segment); } + + // create mutable segment public MutableSegment createMutableSegment(final Configuration conf, CellComparator comparator, long size) { MemStoreLAB memStoreLAB = getMemStoreLAB(conf); return generateMutableSegment(conf, comparator, memStoreLAB, size); } + // create flat immutable segment from skip-list-based old immutable segment + public ImmutableSegment createImmutableSegment(int numOfCells, ImmutableSegment oldSegment, + MemStoreCompactorIterator iterator) + throws IOException { + + // new Segment is using the MSLAB of the previous old Segment, because the MSLAB Chunks + // remain the same in the flattening process + return + new ImmutableSegment( + oldSegment, iterator, oldSegment.getMemStoreLAB(), numOfCells, ClassSize.CELL_ARRAY_ENTRY); + } + + // create new flat immutable segment from compacting old immutable segment + public ImmutableSegment createImmutableSegment( + final Configuration conf, final CellComparator comparator, + int numOfCells, MemStoreCompactorIterator iterator, boolean array) + throws IOException { + MemStoreLAB memStoreLAB = getMemStoreLAB(conf); + return + new ImmutableSegment( + conf, comparator, iterator, memStoreLAB, numOfCells, ClassSize.CELL_ARRAY_ENTRY, array); + } + //****** private methods to instantiate concrete store segments **********// private MutableSegment generateMutableSegment( diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java index 9d7a7230a4da..3aa2dbf109a4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java @@ -51,4 +51,19 @@ public LinkedList getStoreSegments() { public long getVersion() { return version; } + + public int getNumOfCells() { + int totalCells = 0; + for (ImmutableSegment s : storeSegments) { + totalCells += s.getCellsCount(); + } + return totalCells; + } + + public ImmutableSegment getSkipListSegment() { + for (ImmutableSegment s : storeSegments) { + if (!s.hasFlatIndex()) return s; + } + return null; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java index 1a368df9ea96..e2ef413b05be 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java @@ -36,8 +36,8 @@ public class TestCellBlocksSet extends TestCase { private static final int NUM_OF_CELLS = 3; private Cell cells[]; - private CellBlockObjectArray cbOnHeap; - private CellBlockSerialized cbOffHeap; + private CellArrayMap cbOnHeap; + private CellChunkMap cbOffHeap; private final static Configuration conf = new Configuration(); private HeapMemStoreLAB mslab; @@ -47,7 +47,7 @@ public class TestCellBlocksSet extends TestCase { protected void setUp() throws Exception { super.setUp(); - // create array of Cells to bass to the CellBlock under CellSet + // create array of Cells to bass to the CellFlatMap under CellSet final byte[] one = Bytes.toBytes(1); final byte[] two = Bytes.toBytes(2); final byte[] three = Bytes.toBytes(3); @@ -60,7 +60,7 @@ protected void setUp() throws Exception { final KeyValue kv3 = new KeyValue(three, f, q, 30, v); cells = new Cell[] {kv1,kv2,kv3}; - cbOnHeap = new CellBlockObjectArray(CellComparator.COMPARATOR,cells,0,NUM_OF_CELLS,false); + cbOnHeap = new CellArrayMap(CellComparator.COMPARATOR,cells,0,NUM_OF_CELLS,false); conf.setBoolean(SegmentFactory.USEMSLAB_KEY, true); conf.setFloat(MemStoreChunkPool.CHUNK_POOL_MAXSIZE_KEY, 0.2f); @@ -69,18 +69,18 @@ protected void setUp() throws Exception { HeapMemStoreLAB.Chunk[] c = shallowCellsToBuffer(kv1, kv2, kv3); int chunkSize = conf.getInt(HeapMemStoreLAB.CHUNK_SIZE_KEY, HeapMemStoreLAB.CHUNK_SIZE_DEFAULT); - cbOffHeap = new CellBlockSerialized(CellComparator.COMPARATOR, mslab, + cbOffHeap = new CellChunkMap(CellComparator.COMPARATOR, mslab, c, 0, NUM_OF_CELLS, chunkSize, false); } - /* Create and test CellSet based on CellBlockObjectArray */ + /* Create and test CellSet based on CellArrayMap */ public void testCellBlocksOnHeap() throws Exception { CellSet cs = new CellSet(cbOnHeap); testCellBlocks(cs); testIterators(cs); } - /* Create and test CellSet based on CellBlockSerialized */ + /* Create and test CellSet based on CellChunkMap */ public void testCellBlocksOffHeap() throws Exception { CellSet cs = new CellSet(cbOffHeap); testCellBlocks(cs); From 83b2dbda4123d7419cb2b60aaae568af862a106b Mon Sep 17 00:00:00 2001 From: anastas Date: Wed, 6 Apr 2016 16:06:21 +0300 Subject: [PATCH 3/9] For the internal code review --- .../hbase/regionserver/AbstractMemStore.java | 3 +- .../hbase/regionserver/CellFlatMap.java | 17 +- .../regionserver/CompactingMemStore.java | 94 +-- .../hbase/regionserver/HeapMemStoreLAB.java | 8 +- .../hbase/regionserver/ImmutableSegment.java | 63 +- .../hbase/regionserver/MemStoreChunkPool.java | 1 + .../MemStoreCompactorIterator.java | 49 +- .../hbase/regionserver/MutableSegment.java | 2 +- .../hadoop/hbase/regionserver/Segment.java | 8 +- .../regionserver/VersionedSegmentsList.java | 2 +- ...ellBlocksSet.java => TestCellFlatSet.java} | 47 +- .../TestCompactingMemStoreAB.java | 723 ++++++++++++++++++ 12 files changed, 847 insertions(+), 170 deletions(-) rename hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/{TestCellBlocksSet.java => TestCellFlatSet.java} (79%) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStoreAB.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java index 9f94ec0594b6..767551dff423 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java @@ -206,7 +206,8 @@ public long heapSize() { */ @Override public List getScanners(long readPt) throws IOException { - return Collections. singletonList(new MemStoreScanner(this, readPt)); + return Collections. singletonList( + new MemStoreScanner(getComparator(), getListOfScanners(readPt))); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java index 18d30c7847cb..ce7b74f083a7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java @@ -67,7 +67,7 @@ protected abstract CellFlatMap createCellFlatMap(Comparator compar * Positive returned numbers mean the index. * Negative returned numbers means the key not found. * The absolute value of the output is the - * possible insert index for the searched key: (-1 * insertion point) - 1 + * possible insert index for the searched key: (-1 * insertion point) * @param needle The key to look for in all of the entries * @return Same return value as Arrays.binarySearch. */ @@ -77,13 +77,10 @@ private int find(Cell needle) { while (begin <= end) { int mid = begin + ((end - begin) / 2); - Cell midCell = getCellFromIndex(mid); - int compareRes = comparator.compare(midCell, needle); - // 0 means equals. We found the key. - if (compareRes == 0) return mid; + if (compareRes == 0) return mid; // 0 means equals. We found the key else if (compareRes < 0) { // midCell is less than needle so we need to look at farther up begin = mid + 1; @@ -93,14 +90,16 @@ else if (compareRes < 0) { } } - return (-1 * begin) - 1; + return (-1 * begin); } + /* Get the index of the key taking into consideration whether + ** the key should be inclusive or exclusive */ private int getValidIndex(Cell key, boolean inclusive) { int index = find(key); - if (inclusive && index >= 0) index++; - else if (index < 0) index = -(index + 1) - 1; - return index; + if (inclusive && index >= 0) + index = (descending) ? index-1 : index+1; + return Math.abs(index); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index 572f3eda50d4..c7873becf1f0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -58,9 +58,9 @@ public class CompactingMemStore extends AbstractMemStore { public final static long DEEP_OVERHEAD_PER_PIPELINE_FLAT_ARRAY_ITEM = ClassSize.align( ClassSize.TIMERANGE_TRACKER + ClassSize.CELL_SKIPLIST_SET + ClassSize.CELL_ARRAY_MAP); - public final static double IN_MEMORY_FLUSH_THRESHOLD_FACTOR = 0.9; + public final static double COPY_COMPACTION_THRESHOLD_FACTOR = 0.9; public final static double COMPACTION_TRIGGER_REMAIN_FACTOR = 1; - public final static boolean CHECK_FOR_COMPACTION = false; + public final static boolean COMPACTION_PRE_CHECK = false; private static final Log LOG = LogFactory.getLog(CompactingMemStore.class); private HStore store; @@ -301,7 +301,7 @@ void flushInMemory() throws IOException { inMemoryFlushInProgress.set(true); // Speculative compaction execution, may be interrupted if flush is forced while // compaction is in progress - compactor.startCompact(); + compactor.startCompact(0); } } catch (IOException e) { LOG.warn("Unable to run memstore compaction. region " @@ -324,7 +324,7 @@ private ExecutorService getPool() { } private boolean shouldFlushInMemory() { - if(getActive().getSize() > IN_MEMORY_FLUSH_THRESHOLD_FACTOR*flushSizeLowerBound) { + if(getActive().getSize() > COPY_COMPACTION_THRESHOLD_FACTOR *flushSizeLowerBound) { // size above flush threshold return (allowCompaction.get() && !inMemoryFlushInProgress.get()); } @@ -354,6 +354,7 @@ private void pushActiveToPipeline(MutableSegment active) { private void pushTailToSnapshot() { ImmutableSegment tail = pipeline.pullTail(); + if (!tail.isEmpty()) { setSnapshot(tail); long size = getSegmentSize(tail); @@ -397,9 +398,6 @@ public void process() throws IOException { */ private class MemStoreCompactor { - // list of Scanners of segments in the pipeline, when compaction starts - List scanners = new ArrayList(); - // a static version of the segment list from the pipeline private VersionedSegmentsList versionedList; @@ -417,25 +415,16 @@ private class MemStoreCompactor { * * is already an ongoing compaction (or pipeline is empty). */ - public boolean startCompact() throws IOException { - - //org.junit.Assert.assertTrue("\n\n<<< Starting compaction\n", false); + public boolean startCompact(int i) throws IOException { if (pipeline.isEmpty()) return false; // no compaction on empty pipeline // get the list of segments from the pipeline, the list is marked with specific version versionedList = pipeline.getVersionedList(); - //org.junit.Assert.assertTrue("\n\n<<< Starting compaction, got versioned list\n", false); - - // create the list of scanners with maximally possible read point, meaning that - // all KVs are going to be returned by the pipeline traversing - for (Segment segment : versionedList.getStoreSegments()) { - scanners.add(segment.getSegmentScanner(Long.MAX_VALUE)); - } LOG.info( "Starting the MemStore in-memory compaction for store " + store.getColumnFamilyName()); - //org.junit.Assert.assertTrue("\n\n<<< Starting actual compaction\n", false); - doCompact(); + + doCompact(i); return true; } @@ -463,54 +452,43 @@ private void releaseResources() { * The solo (per compactor) thread only reads the compaction pipeline. * There is at most one thread per memstore instance. */ - private void doCompact() { + private void doCompact(int y) { int cellsAfterComp = versionedList.getNumOfCells(); - ImmutableSegment result = null; - - //org.junit.Assert.assertTrue("\n\n<<< Starting doCompact() 1\n", false); - try { // the compaction processing - - if (CHECK_FOR_COMPACTION) { - //org.junit.Assert.assertTrue("\n\n<<< Starting doCompact() 2\n", false); - // Phase I: estimate the compaction expedience - CHECK COMPACTION + // Phase I: estimate the compaction expedience - CHECK COMPACTION + if (COMPACTION_PRE_CHECK) { cellsAfterComp = countCellsForCompaction(); - // org.junit.Assert.assertTrue("\n\n<<< In compaction, the number of cells after " - // + "compaction should be: " + cellsAfterComp + " \n", false); if (!isInterrupted.get() && (cellsAfterComp > COMPACTION_TRIGGER_REMAIN_FACTOR * versionedList.getNumOfCells())) { // too much cells "survive" the possible compaction we do not want to compact! LOG.debug("Stopping the unworthy MemStore in-memory compaction for store " + getFamilyName()); - // Looking for Segment in pipeline snapshot with SkipList index, to make it flat + // Looking for Segment in pipeline with SkipList index, to make it flat ImmutableSegment segment = versionedList.getSkipListSegment(); if (segment == null) { Thread.currentThread().interrupt(); return; } - // We are going to flatten the given segment with SkipList index + // Flatten the given segment with SkipList index and replace it in the pipeline flattenSegment(segment, cellsAfterComp); return; } } -// org.junit.Assert.assertTrue("\n\n<<< In compaction, decided to compact! " -// + "The number of cells after " + "compaction should be: " + cellsAfterComp + " \n", false); - + ImmutableSegment result = null; // Phase II: create the compacted ImmutableSegment - START COMPACTION if (!isInterrupted.get()) { switch (type) { - case COMPACT_TO_SKIPLIST_MAP: result = compactToNotFlatSegment(); - case COMPACT_TO_ARRAY_MAP: result = compactToFlatSegment(cellsAfterComp, true); - case COMPACT_TO_CHUNK_MAP: result = compactToFlatSegment(cellsAfterComp, false); - default: assert false; // sanity check + case COMPACT_TO_SKIPLIST_MAP: result = compactToNotFlatSegment(cellsAfterComp); + break; + case COMPACT_TO_ARRAY_MAP: result = compactToFlatSegment(cellsAfterComp, true); + break; + case COMPACT_TO_CHUNK_MAP: result = compactToFlatSegment(cellsAfterComp, false); + break; + default: // sanity check + org.junit.Assert.assertTrue("\n\n<<< Wrong compacting MemStore type", false); } } - -// org.junit.Assert.assertTrue("\n\n<<< After compaction! " -// + "The number of cells after " + "compaction should be: " + cellsAfterComp -// + " \n", false); - // Phase III: swap the old compaction pipeline - END COMPACTION if (!isInterrupted.get()) { pipeline.swap(versionedList, result); @@ -518,11 +496,9 @@ private void doCompact() { updateLowestUnflushedSequenceIdInWal(true); // only if greater } } catch (Exception e) { - org.junit.Assert.assertTrue("\n\n<<< GOT EXCEPTION IN COMPACTION!!! " + e + "\n", false); LOG.debug("Interrupting the MemStore in-memory compaction for store " + getFamilyName()); Thread.currentThread().interrupt(); } finally { - //org.junit.Assert.assertTrue("\n\n<<< BEFORE RELEASE RESOURCE!!! " + "\n", false); releaseResources(); inMemoryFlushInProgress.set(false); } @@ -538,7 +514,7 @@ private void flattenSegment(ImmutableSegment segment, int numOfCells) throws IOE LOG.debug("Flattening Segment " + segment); // Update the pipeline scanner (list of scanners) to scan only Skip List based Segment List scanners = new ArrayList(); - scanners.add(segment.getSegmentScanner(Long.MAX_VALUE)); + scanners.add(segment.getSegmentScanner(store.getSmallestReadPoint())); ImmutableSegment newSegment = createFlatFromSegmentWithSkipList(numOfCells, segment); pipeline.replace(versionedList, segment, newSegment); @@ -546,7 +522,7 @@ private void flattenSegment(ImmutableSegment segment, int numOfCells) throws IOE } /** - * COMPACT TO ARRAY MAP + * COMPACT TO ARRAY MAP OR TO CHUNK MAP * Updates the given single Segment using the internal store scanner, * who in turn uses ScanQueryMatcher */ @@ -555,9 +531,8 @@ private ImmutableSegment compactToFlatSegment(int numOfCells, boolean array) MemStoreCompactorIterator iterator = new MemStoreCompactorIterator( - 2, versionedList.getStoreSegments(), getComparator(), compactionKVMax, store); + numOfCells, versionedList.getStoreSegments(), getComparator(), compactionKVMax, store); -// org.junit.Assert.assertTrue("\n\n<<< After creating the scanner in compaction\n", false); ImmutableSegment result = SegmentFactory.instance() .createImmutableSegment(getConfiguration(), getComparator(), numOfCells, iterator, array); @@ -568,17 +543,15 @@ private ImmutableSegment compactToFlatSegment(int numOfCells, boolean array) /** * COMPACT TO SKIPLIST MAP */ - private ImmutableSegment compactToNotFlatSegment() throws IOException { + private ImmutableSegment compactToNotFlatSegment(int cells) throws IOException { ImmutableSegment result = SegmentFactory.instance() // create the scanner .createImmutableSegment(getConfiguration(), getComparator(), CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_ITEM); - //org.junit.Assert.assertTrue("\n\n<<< Starting counting cells for comaction 1\n", false); MemStoreCompactorIterator iterator = new MemStoreCompactorIterator( - 3,versionedList.getStoreSegments(),getComparator(),compactionKVMax, store); + cells,versionedList.getStoreSegments(),getComparator(),compactionKVMax, store); - //org.junit.Assert.assertTrue("\n\n<<< Starting counting cells for comaction 2\n", false); while (iterator.hasNext()) { Cell c = iterator.next(); // The scanner is doing all the elimination logic @@ -588,8 +561,6 @@ private ImmutableSegment compactToNotFlatSegment() throws IOException { result.internalAdd(newKV); } iterator.remove(); - // org.junit.Assert.assertTrue("\n\n<<< Finished counting " - // + cnt + " cells for comaction\n", false); return result; } @@ -615,19 +586,14 @@ private ImmutableSegment createFlatFromSegmentWithSkipList(int numOfCells, private int countCellsForCompaction() throws IOException { - //org.junit.Assert.assertTrue("\n\n<<< Starting counting cells for comaction 1\n", false); MemStoreCompactorIterator iterator = new MemStoreCompactorIterator( 3,versionedList.getStoreSegments(),getComparator(),compactionKVMax, store); int cnt = 0; - - //org.junit.Assert.assertTrue("\n\n<<< Starting counting cells for comaction 2\n", false); while (iterator.next() != null) { cnt++; } iterator.remove(); -// org.junit.Assert.assertTrue("\n\n<<< Finished counting " -// + cnt + " cells for comaction\n", false); return cnt; } } // end of the MemStoreCompactor Class @@ -665,10 +631,10 @@ Cell getNextRow(final Cell cell) { return lowest; } - // debug method - private void debug() { + // debug method, also for testing + public void debug() { String msg = "active size="+getActive().getSize(); - msg += " threshold="+IN_MEMORY_FLUSH_THRESHOLD_FACTOR*flushSizeLowerBound; + msg += " threshold="+ COPY_COMPACTION_THRESHOLD_FACTOR *flushSizeLowerBound; msg += " allow compaction is "+ (allowCompaction.get() ? "true" : "false"); msg += " inMemoryFlushInProgress is "+ (inMemoryFlushInProgress.get() ? "true" : "false"); LOG.debug(msg); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java index 05b5febf250b..00df16423050 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java @@ -192,13 +192,10 @@ private Chunk getOrMakeChunk() { // No current chunk, so we want to allocate one. We race // against other allocators to CAS in an uninitialized chunk // (which is cheap to allocate) - - //c = (chunkPool != null) ? chunkPool.getChunk() : new Chunk(chunkSize, 5); //HBASE-14921 - if(chunkPool != null) { c = chunkPool.getChunk(); } else { - // HBASE-14921 555 is here till it is decided whether ChunkPool is always on + // HBASE-14921: 555 is here till it is decided whether ChunkPool is always on c = new Chunk(chunkSize, 555); c.init(); } @@ -207,7 +204,6 @@ private Chunk getOrMakeChunk() { // we won race - now we need to actually do the expensive // allocation step - //c.init(); //14921 this.chunkQueue.add(c); return c; } else if (chunkPool != null) { @@ -303,8 +299,6 @@ public void init() { // We should always succeed the above CAS since only one thread // calls init()! Preconditions.checkState(initted, "Multiple threads tried to init same chunk"); - - //org.junit.Assert.assertTrue("\n\n inside chunk initialization 3", false); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java index 4d2c67cc2d5c..64bebd58be2d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java @@ -39,53 +39,56 @@ public class ImmutableSegment extends Segment { private boolean isFlat; // whether it is based on CellFlatMap or ConcurrentSkipListMap + /** + * Builds a special scanner for the MemStoreSnapshot object that is different than the + * general segment scanner. + * @return a special scanner for the MemStoreSnapshot object + */ + public KeyValueScanner getKeyValueScanner() { + return new CollectionBackedScanner(getCellSet(), getComparator()); + } + + public boolean isFlat() { + return isFlat; + } + protected ImmutableSegment(Segment segment) { super(segment); isFlat = false; } - // flattening + // C-tor by flattening other (old) segment protected ImmutableSegment(ImmutableSegment oldSegment, MemStoreCompactorIterator iterator, MemStoreLAB memStoreLAB, int numOfCells, long constantCellSizeOverhead) { super(null,oldSegment.getComparator(),memStoreLAB, CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_FLAT_ARRAY_ITEM, constantCellSizeOverhead); - - // build the CellSet - CellSet cs = this.createArrayBasedCellSet(numOfCells, iterator, false); - - // update the CellSet of the new Segment - this.setCellSet(cs); + CellSet cs = this.createArrayBasedCellSet(numOfCells, iterator, false); // build the CellSet + this.setCellSet(cs); // update the CellSet of the new Segment isFlat = true; } - // compaction to CellArrayMap + // C-tor by compaction to CellArrayMap or CellChunkMap protected ImmutableSegment( final Configuration conf, CellComparator comparator, MemStoreCompactorIterator iterator, MemStoreLAB memStoreLAB, int numOfCells, long constantCellSizeOverhead, boolean array) { super(null, comparator, memStoreLAB, CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_FLAT_ARRAY_ITEM, constantCellSizeOverhead); - -// org.junit.Assert.assertTrue("\n\n<<< Immutable Segment Constructor BEFORE compaction\n", false); - - // build the CellSet - CellSet cs = null; + CellSet cs = null; // build the CellSet Cell array or Byte array based if (array) { cs = this.createArrayBasedCellSet(numOfCells, iterator, true); } else { cs = this.createChunkBasedCellSet(numOfCells, iterator, conf); } - - // update the CellSet of the new Segment - this.setCellSet(cs); + this.setCellSet(cs); // update the CellSet of the new Segment isFlat = true; } - private CellSet createArrayBasedCellSet(int numOfCells, MemStoreCompactorIterator iterator, - boolean allocateFromMSLAB) { - // build the Cell Array - Cell[] cells = new Cell[numOfCells]; + private CellSet createArrayBasedCellSet( + int numOfCells, MemStoreCompactorIterator iterator, boolean allocateFromMSLAB) { + + Cell[] cells = new Cell[numOfCells]; // build the Cell Array int i = 0; while (iterator.hasNext()) { Cell c = iterator.next(); @@ -95,18 +98,17 @@ private CellSet createArrayBasedCellSet(int numOfCells, MemStoreCompactorIterato KeyValue kv = KeyValueUtil.ensureKeyValue(c); Cell newKV = maybeCloneWithAllocator(kv); cells[i++] = newKV; - // flattening = !allocateFromMSLAB (false), counting both Heap and MetaData size + // flattening = false, compaction case, counting both Heap and MetaData size updateMetaInfo(c,true,!allocateFromMSLAB); } else { cells[i++] = c; - // flattening = !allocateFromMSLAB (true), counting only MetaData size + // flattening = true, flattening case, counting only MetaData size updateMetaInfo(c,true,!allocateFromMSLAB); } } // build the immutable CellSet - CellArrayMap - cam = new CellArrayMap(getComparator(),cells,0,numOfCells,false); + CellArrayMap cam = new CellArrayMap(getComparator(),cells,0,i,false); return new CellSet(cam); } @@ -147,19 +149,6 @@ private CellSet createChunkBasedCellSet( return new CellSet(ccm); } - /** - * Builds a special scanner for the MemStoreSnapshot object that is different than the - * general segment scanner. - * @return a special scanner for the MemStoreSnapshot object - */ - public KeyValueScanner getKeyValueScanner() { - return new CollectionBackedScanner(getCellSet(), getComparator()); - } - - public boolean isFlat() { - return isFlat; - } - /** * If the segment has a memory allocator the cell is being cloned to this space, and returned; * otherwise the given cell is returned diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java index cf2231e2f675..32f3f034cf48 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java @@ -94,6 +94,7 @@ public class MemStoreChunkPool { for (int i = 0; i < initialCount; i++) { Chunk chunk = allocateChunk(); + reclaimedChunks.add(chunk); } final String n = Thread.currentThread().getName(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorIterator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorIterator.java index 6c813a0dd605..506671d61213 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorIterator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorIterator.java @@ -37,13 +37,12 @@ public class MemStoreCompactorIterator implements Iterator { private List kvs = new ArrayList(); - private int c = 0; + // scanner for full or partial pipeline (heap of segment scanners) private KeyValueScanner scanner; + // scanner on top of pipeline scanner that uses ScanQueryMatcher private StoreScanner compactingScanner; - // get the limit to the size of the groups to be returned by compactingScanner - private final int compactionKVMax; private final ScannerContext scannerContext; @@ -53,10 +52,8 @@ public class MemStoreCompactorIterator implements Iterator { // C-tor public MemStoreCompactorIterator(int initiator, LinkedList segments, CellComparator comparator, int compactionKVMax, HStore store) throws IOException { - this.compactionKVMax = compactionKVMax; - this.scannerContext = - ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build(); - //org.junit.Assert.assertTrue("\n\n<<< Initiating compactor iterator\n", false); + + this.scannerContext = ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build(); // list of Scanners of segments in the pipeline, when compaction starts List scanners = new ArrayList(); @@ -64,46 +61,44 @@ public MemStoreCompactorIterator(int initiator, LinkedList seg // create the list of scanners with maximally possible read point, meaning that // all KVs are going to be returned by the pipeline traversing for (Segment segment : segments) { - scanners.add(segment.getSegmentScanner(Long.MAX_VALUE)); + scanners.add(segment.getSegmentScanner(store.getSmallestReadPoint())); } - scanner = - new MemStoreScanner(comparator, scanners, MemStoreScanner.Type.COMPACT_FORWARD); + scanner = new MemStoreScanner(comparator, scanners, MemStoreScanner.Type.COMPACT_FORWARD); // reinitialize the compacting scanner for each instance of iterator compactingScanner = createScanner(store, scanner, initiator); hasMore = compactingScanner.next(kvs, scannerContext); + if (!kvs.isEmpty()) { kvsIterator = kvs.iterator(); } -// if (initiator == 2) -// org.junit.Assert.assertTrue("\n\n<<< Iterator constructing. " -// + "Store: " + store + ", Scanner: " + scanner + ", hasMore: " + hasMore + "\n", false); } @Override public boolean hasNext() { if (!kvsIterator.hasNext()) { - if (!refillKVS()) return false; + if (!refillKVS()) { + return false; + } } return hasMore; } + + @Override public Cell next() { - c++; - //if (c>1) org.junit.Assert.assertTrue("\n\n<<< " + c + " iterations of iterator\n",false); if (!kvsIterator.hasNext()) { - // org.junit.Assert.assertTrue("\n\n<<< Got to empty kvs for the first time with " + c - // + " iterations\n", false); if (!refillKVS()) return null; } - return (!hasMore) ? null : kvsIterator.next(); } + + @Override public void remove() { compactingScanner.close(); @@ -112,6 +107,8 @@ public void remove() { scanner = null; } + + /** * Creates the scanner for compacting the pipeline. * @@ -122,10 +119,6 @@ private StoreScanner createScanner(Store store, KeyValueScanner scanner, int ini Scan scan = new Scan(); scan.setMaxVersions(); //Get all available versions - -// if (init == 2) -// org.junit.Assert.assertTrue("\n\n<<< Before creating the internal scanner " + "\n", false); - StoreScanner internalScanner = new StoreScanner(store, store.getScanInfo(), scan, Collections.singletonList(scanner), ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), @@ -134,19 +127,17 @@ private StoreScanner createScanner(Store store, KeyValueScanner scanner, int ini return internalScanner; } + + private boolean refillKVS() { kvs.clear(); // clear previous KVS, first initiated in the constructor - if (!hasMore) { // if there is nothing expected next in compactingScanner - org.junit.Assert.assertTrue("\n\n<<< Got to empty kvs for the first time with " + c - + " iterations and no more\n", false); return false; } try { // try to get next KVS hasMore = compactingScanner.next(kvs, scannerContext); } catch (IOException ie) { - org.junit.Assert.assertTrue("\n\n<<< GOT EXCEPTION IN ITERATOR !!!\n",false); throw new IllegalStateException(ie); } @@ -154,10 +145,10 @@ private boolean refillKVS() { kvsIterator = kvs.iterator(); return true; } else { - // org.junit.Assert.assertTrue("\n\n<<< Got to empty kvs for the first time with " + c - // + " iterations and empty set\n", false); return false; } } + + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java index 73766261f0db..da31167ea0aa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java @@ -49,7 +49,7 @@ public long add(Cell cell) { public long rollback(Cell cell) { Cell found = getCellSet().get(cell); if (found != null && found.getSequenceId() == cell.getSequenceId()) { - long sz = AbstractMemStore.heapSizeChange(cell, true, false); + long sz = heapSizeChange(cell, true); getCellSet().remove(cell); incSize(-sz); return sz; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java index 1e0c5a92a5cd..63db3b5e0a98 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java @@ -47,7 +47,6 @@ public abstract class Segment { private final TimeRangeTracker timeRangeTracker; private long constantCellMetaDataSize; protected volatile boolean tagsPresent; - protected boolean hasFlatIndex; protected Segment( CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, long size, @@ -59,7 +58,6 @@ protected Segment( this.size = new AtomicLong(size); this.timeRangeTracker = new TimeRangeTracker(); this.tagsPresent = false; - this.hasFlatIndex = true; this.constantCellMetaDataSize = constantCellSize; } @@ -160,10 +158,6 @@ public boolean isTagsPresent() { return tagsPresent; } - public boolean hasFlatIndex() { - return hasFlatIndex; - } - public void incScannerCount() { if(getMemStoreLAB() != null) { getMemStoreLAB().incScannerCount(); @@ -192,7 +186,7 @@ public Segment setSize(long size) { * @return this object */ - public Segment setCellSet(CellSet cellSet) { + protected Segment setCellSet(CellSet cellSet) { this.cellSet = cellSet; return this; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java index 3aa2dbf109a4..c622a98609b0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java @@ -62,7 +62,7 @@ public int getNumOfCells() { public ImmutableSegment getSkipListSegment() { for (ImmutableSegment s : storeSegments) { - if (!s.hasFlatIndex()) return s; + if (!s.isFlat()) return s; } return null; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java similarity index 79% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java index e2ef413b05be..e80c030b1fd5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java @@ -27,13 +27,14 @@ import org.junit.experimental.categories.Category; import java.util.Iterator; +import java.util.NavigableMap; import java.util.SortedSet; import static org.junit.Assert.assertTrue; @Category({RegionServerTests.class, SmallTests.class}) -public class TestCellBlocksSet extends TestCase { +public class TestCellFlatSet extends TestCase { - private static final int NUM_OF_CELLS = 3; + private static final int NUM_OF_CELLS = 4; private Cell cells[]; private CellArrayMap cbOnHeap; @@ -43,14 +44,15 @@ public class TestCellBlocksSet extends TestCase { private HeapMemStoreLAB mslab; - protected void setUp() throws Exception { super.setUp(); // create array of Cells to bass to the CellFlatMap under CellSet - final byte[] one = Bytes.toBytes(1); - final byte[] two = Bytes.toBytes(2); - final byte[] three = Bytes.toBytes(3); + final byte[] one = Bytes.toBytes(15); + final byte[] two = Bytes.toBytes(25); + final byte[] three = Bytes.toBytes(35); + final byte[] four = Bytes.toBytes(45); + final byte[] f = Bytes.toBytes("f"); final byte[] q = Bytes.toBytes("q"); final byte[] v = Bytes.toBytes(4); @@ -58,8 +60,9 @@ protected void setUp() throws Exception { final KeyValue kv1 = new KeyValue(one, f, q, 10, v); final KeyValue kv2 = new KeyValue(two, f, q, 20, v); final KeyValue kv3 = new KeyValue(three, f, q, 30, v); + final KeyValue kv4 = new KeyValue(four, f, q, 40, v); - cells = new Cell[] {kv1,kv2,kv3}; + cells = new Cell[] {kv1,kv2,kv3,kv4}; cbOnHeap = new CellArrayMap(CellComparator.COMPARATOR,cells,0,NUM_OF_CELLS,false); conf.setBoolean(SegmentFactory.USEMSLAB_KEY, true); @@ -67,7 +70,7 @@ protected void setUp() throws Exception { MemStoreChunkPool.chunkPoolDisabled = false; mslab = new HeapMemStoreLAB(conf); - HeapMemStoreLAB.Chunk[] c = shallowCellsToBuffer(kv1, kv2, kv3); + HeapMemStoreLAB.Chunk[] c = shallowCellsToBuffer(kv1, kv2, kv3, kv4); int chunkSize = conf.getInt(HeapMemStoreLAB.CHUNK_SIZE_KEY, HeapMemStoreLAB.CHUNK_SIZE_DEFAULT); cbOffHeap = new CellChunkMap(CellComparator.COMPARATOR, mslab, c, 0, NUM_OF_CELLS, chunkSize, false); @@ -89,9 +92,16 @@ public void testCellBlocksOffHeap() throws Exception { /* Generic basic test for immutable CellSet */ private void testCellBlocks(CellSet cs) throws Exception { - assertEquals(NUM_OF_CELLS, cs.size()); // check size + final byte[] oneAndHalf = Bytes.toBytes(20); + final byte[] f = Bytes.toBytes("f"); + final byte[] q = Bytes.toBytes("q"); + final byte[] v = Bytes.toBytes(4); + final KeyValue outerCell = new KeyValue(oneAndHalf, f, q, 10, v); - assertTrue(cs.contains(cells[0])); // check existance of the first + assertEquals(NUM_OF_CELLS, cs.size()); // check size + assertFalse(cs.contains(outerCell)); // check outer cell + + assertTrue(cs.contains(cells[0])); // check existence of the first Cell first = cs.first(); assertTrue(cells[0].equals(first)); @@ -99,15 +109,18 @@ private void testCellBlocks(CellSet cs) throws Exception { Cell last = cs.last(); assertTrue(cells[NUM_OF_CELLS - 1].equals(last)); - SortedSet tail = cs.tailSet(cells[1]); // check tail abd head sizes - assertEquals(2, tail.size()); + SortedSet tail = cs.tailSet(cells[1]); // check tail abd head sizes + assertEquals(NUM_OF_CELLS - 1, tail.size()); SortedSet head = cs.headSet(cells[1]); assertEquals(1, head.size()); + SortedSet tailOuter = cs.tailSet(outerCell); // check tail starting from outer cell + assertEquals(NUM_OF_CELLS - 1, tailOuter.size()); + Cell tailFirst = tail.first(); assertTrue(cells[1].equals(tailFirst)); Cell tailLast = tail.last(); - assertTrue(cells[2].equals(tailLast)); + assertTrue(cells[NUM_OF_CELLS - 1].equals(tailLast)); Cell headFirst = head.first(); assertTrue(cells[0].equals(headFirst)); @@ -142,7 +155,7 @@ private void testIterators(CellSet cs) throws Exception { } /* Create byte array holding shallow Cells referencing to the deep Cells data */ - private HeapMemStoreLAB.Chunk[] shallowCellsToBuffer(Cell kv1, Cell kv2, Cell kv3) { + private HeapMemStoreLAB.Chunk[] shallowCellsToBuffer(Cell kv1, Cell kv2, Cell kv3, Cell kv4) { HeapMemStoreLAB.Chunk chunkD = mslab.allocateChunk(); HeapMemStoreLAB.Chunk chunkS = mslab.allocateChunk(); HeapMemStoreLAB.Chunk result[] = {chunkS}; @@ -168,6 +181,12 @@ private HeapMemStoreLAB.Chunk[] shallowCellsToBuffer(Cell kv1, Cell kv2, Cell kv pos = Bytes.putInt(shallowBuffer, pos, chunkD.getId()); // deep chunk index pos = Bytes.putInt(shallowBuffer, pos, offset); // offset pos = Bytes.putInt(shallowBuffer, pos, KeyValueUtil.length(kv3)); // length + offset += KeyValueUtil.length(kv3); + + KeyValueUtil.appendToByteArray(kv4, deepBuffer, offset); // write deep cell data + pos = Bytes.putInt(shallowBuffer, pos, chunkD.getId()); // deep chunk index + pos = Bytes.putInt(shallowBuffer, pos, offset); // offset + pos = Bytes.putInt(shallowBuffer, pos, KeyValueUtil.length(kv4)); // length return result; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStoreAB.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStoreAB.java new file mode 100644 index 000000000000..c91ea4085d01 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStoreAB.java @@ -0,0 +1,723 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.util.ArrayList; +import java.util.List; + +/** + * compacted memstore test case + */ +@Category({RegionServerTests.class, MediumTests.class}) +public class TestCompactingMemStoreAB extends TestCompactingMemStore { + + private static final Log LOG = LogFactory.getLog(TestCompactingMemStoreAB.class); + private static MemStoreChunkPool chunkPool; + private HRegion region; + private RegionServicesForStores regionServicesForStores; + private HStore store; + + ////////////////////////////////////////////////////////////////////////////// + // Helpers + ////////////////////////////////////////////////////////////////////////////// + private static byte[] makeQualifier(final int i1, final int i2) { + return Bytes.toBytes(Integer.toString(i1) + ";" + + Integer.toString(i2)); + } + + @Override public void tearDown() throws Exception { + chunkPool.clearChunks(); + } + + @Override public void setUp() throws Exception { + super.internalSetUp(); + Configuration conf = new Configuration(); + conf.setBoolean(SegmentFactory.USEMSLAB_KEY, true); + conf.setFloat(MemStoreChunkPool.CHUNK_POOL_MAXSIZE_KEY, 0.2f); + conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000); + HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf); + HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); + this.region = hbaseUtility.createTestRegion("foobar", hcd); + this.regionServicesForStores = region.getRegionServicesForStores(); + this.store = new HStore(region, hcd, conf); + this.memstore = + new CompactingMemStore(HBaseConfiguration.create(), CellComparator.COMPARATOR, store, + regionServicesForStores, CompactingMemStore.Type.COMPACT_TO_ARRAY_MAP); + chunkPool = MemStoreChunkPool.getPool(conf); + assertTrue(chunkPool != null); + } + + /** + * A simple test which verifies the 3 possible states when scanning across snapshot. + * + * @throws IOException + * @throws CloneNotSupportedException + */ + public void testScanAcrossSnapshot2() throws IOException, CloneNotSupportedException { + // we are going to the scanning across snapshot with two kvs + // kv1 should always be returned before kv2 + final byte[] one = Bytes.toBytes(1); + final byte[] two = Bytes.toBytes(2); + final byte[] f = Bytes.toBytes("f"); + final byte[] q = Bytes.toBytes("q"); + final byte[] v = Bytes.toBytes(3); + + final KeyValue kv1 = new KeyValue(one, f, q, 10, v); + final KeyValue kv2 = new KeyValue(two, f, q, 10, v); + + // use case 1: both kvs in kvset + this.memstore.add(kv1.clone()); + this.memstore.add(kv2.clone()); + verifyScanAcrossSnapshot2(kv1, kv2); + + // use case 2: both kvs in snapshot + this.memstore.snapshot(0); + verifyScanAcrossSnapshot2(kv1, kv2); + + // use case 3: first in snapshot second in kvset + this.memstore = + new CompactingMemStore(HBaseConfiguration.create(), CellComparator.COMPARATOR, store, + regionServicesForStores); + this.memstore.add(kv1.clone()); + // As compaction is starting in the background the repetition + // of the k1 might be removed BUT the scanners created earlier + // should look on the OLD MutableCellSetSegment, so this should be OK... + this.memstore.snapshot(0); + this.memstore.add(kv2.clone()); + verifyScanAcrossSnapshot2(kv1, kv2); + } + + /** + * Test memstore snapshots + * + * @throws IOException + */ + public void testSnapshotting() throws IOException { + final int snapshotCount = 5; + // Add some rows, run a snapshot. Do it a few times. + for (int i = 0; i < snapshotCount; i++) { + addRows(this.memstore); + runSnapshot(this.memstore, true); + assertEquals("History not being cleared", 0, this.memstore.getSnapshot().getCellsCount()); + } + } + + ////////////////////////////////////////////////////////////////////////////// + // Get tests + ////////////////////////////////////////////////////////////////////////////// + + /** + * Test getNextRow from memstore + * + * @throws InterruptedException + */ + public void testGetNextRow() throws Exception { + addRows(this.memstore); + // Add more versions to make it a little more interesting. + Thread.sleep(1); + addRows(this.memstore); + Cell closestToEmpty = ((CompactingMemStore) this.memstore).getNextRow(KeyValue.LOWESTKEY); + assertTrue(KeyValue.COMPARATOR + .compareRows(closestToEmpty, new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0); + for (int i = 0; i < ROW_COUNT; i++) { + Cell nr = ((CompactingMemStore) this.memstore) + .getNextRow(new KeyValue(Bytes.toBytes(i), System.currentTimeMillis())); + if (i + 1 == ROW_COUNT) { + assertEquals(nr, null); + } else { + assertTrue(KeyValue.COMPARATOR + .compareRows(nr, new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0); + } + } + //starting from each row, validate results should contain the starting row + Configuration conf = HBaseConfiguration.create(); + for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) { + ScanInfo scanInfo = + new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE, 0, this.memstore.getComparator()); + ScanType scanType = ScanType.USER_SCAN; + InternalScanner scanner = + new StoreScanner(new Scan(Bytes.toBytes(startRowId)), scanInfo, scanType, null, memstore.getScanners(0)); + List results = new ArrayList(); + for (int i = 0; scanner.next(results); i++) { + int rowId = startRowId + i; + Cell left = results.get(0); + byte[] row1 = Bytes.toBytes(rowId); + assertTrue("Row name", + CellComparator.COMPARATOR.compareRows(left, row1, 0, row1.length) == 0); + assertEquals("Count of columns", QUALIFIER_COUNT, results.size()); + List row = new ArrayList(); + for (Cell kv : results) { + row.add(kv); + } + isExpectedRowWithoutTimestamps(rowId, row); + // Clear out set. Otherwise row results accumulate. + results.clear(); + } + } + } + + public void testGet_memstoreAndSnapShot() throws IOException { + byte[] row = Bytes.toBytes("testrow"); + byte[] fam = Bytes.toBytes("testfamily"); + byte[] qf1 = Bytes.toBytes("testqualifier1"); + byte[] qf2 = Bytes.toBytes("testqualifier2"); + byte[] qf3 = Bytes.toBytes("testqualifier3"); + byte[] qf4 = Bytes.toBytes("testqualifier4"); + byte[] qf5 = Bytes.toBytes("testqualifier5"); + byte[] val = Bytes.toBytes("testval"); + + //Setting up memstore + memstore.add(new KeyValue(row, fam, qf1, val)); + memstore.add(new KeyValue(row, fam, qf2, val)); + memstore.add(new KeyValue(row, fam, qf3, val)); + //Pushing to pipeline + ((CompactingMemStore) memstore).flushInMemory(); + assertEquals(0, memstore.getSnapshot().getCellsCount()); + //Creating a snapshot + memstore.snapshot(); + assertEquals(3, memstore.getSnapshot().getCellsCount()); + //Adding value to "new" memstore + assertEquals(0, memstore.getActive().getCellsCount()); + memstore.add(new KeyValue(row, fam, qf4, val)); + memstore.add(new KeyValue(row, fam, qf5, val)); + assertEquals(2, memstore.getActive().getCellsCount()); + } + + //////////////////////////////////// + //Test for upsert with MSLAB + //////////////////////////////////// + + /** + * Test a pathological pattern that shows why we can't currently + * use the MSLAB for upsert workloads. This test inserts data + * in the following pattern: + * - row0001 through row1000 (fills up one 2M Chunk) + * - row0002 through row1001 (fills up another 2M chunk, leaves one reference + * to the first chunk + * - row0003 through row1002 (another chunk, another dangling reference) + * This causes OOME pretty quickly if we use MSLAB for upsert + * since each 2M chunk is held onto by a single reference. + */ + public void testUpsertMSLAB() throws Exception { + + int ROW_SIZE = 2048; + byte[] qualifier = new byte[ROW_SIZE - 4]; + + MemoryMXBean bean = ManagementFactory.getMemoryMXBean(); + for (int i = 0; i < 3; i++) { + System.gc(); + } + long usageBefore = bean.getHeapMemoryUsage().getUsed(); + + long size = 0; + long ts = 0; + + for (int newValue = 0; newValue < 1000; newValue++) { + for (int row = newValue; row < newValue + 1000; row++) { + byte[] rowBytes = Bytes.toBytes(row); + size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts); + } + } + System.out.println("Wrote " + ts + " vals"); + for (int i = 0; i < 3; i++) { + System.gc(); + } + long usageAfter = bean.getHeapMemoryUsage().getUsed(); + System.out.println( + "Memory used: " + (usageAfter - usageBefore) + " (heapsize: " + memstore.heapSize() + + " size: " + size + ")"); + } + + //////////////////////////////////// + // Test for periodic memstore flushes + // based on time of oldest edit + //////////////////////////////////// + + /** + * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased + * as older keyvalues are deleted from the memstore. + * + * @throws Exception + */ + public void testUpsertMemstoreSize() throws Exception { + long oldSize = memstore.size(); + + List l = new ArrayList(); + KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v"); + KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v"); + KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v"); + + kv1.setSequenceId(1); + kv2.setSequenceId(1); + kv3.setSequenceId(1); + l.add(kv1); + l.add(kv2); + l.add(kv3); + + this.memstore.upsert(l, 2);// readpoint is 2 + long newSize = this.memstore.size(); + assert (newSize > oldSize); + //The kv1 should be removed. + assert (memstore.getActive().getCellsCount() == 2); + + KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v"); + kv4.setSequenceId(1); + l.clear(); + l.add(kv4); + this.memstore.upsert(l, 3); + assertEquals(newSize, this.memstore.size()); + //The kv2 should be removed. + assert (memstore.getActive().getCellsCount() == 2); + //this.memstore = null; + } + + /** + * Tests that the timeOfOldestEdit is updated correctly for the + * various edit operations in memstore. + * + * @throws Exception + */ + public void testUpdateToTimeOfOldestEdit() throws Exception { + try { + EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest(); + EnvironmentEdgeManager.injectEdge(edge); + long t = memstore.timeOfOldestEdit(); + assertEquals(t, Long.MAX_VALUE); + + // test the case that the timeOfOldestEdit is updated after a KV add + memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v")); + t = memstore.timeOfOldestEdit(); + assertTrue(t == 1234); + // The method will also assert + // the value is reset to Long.MAX_VALUE + t = runSnapshot(memstore, true); + + // test the case that the timeOfOldestEdit is updated after a KV delete + memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v")); + t = memstore.timeOfOldestEdit(); + assertTrue(t == 1234); + t = runSnapshot(memstore, true); + + // test the case that the timeOfOldestEdit is updated after a KV upsert + List l = new ArrayList(); + KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v"); + kv1.setSequenceId(100); + l.add(kv1); + memstore.upsert(l, 1000); + t = memstore.timeOfOldestEdit(); + assertTrue(t == 1234); + } finally { + EnvironmentEdgeManager.reset(); + } + } + + private long runSnapshot(final AbstractMemStore hmc, boolean useForce) throws IOException { + // Save off old state. + long oldHistorySize = hmc.getSnapshot().getSize(); + long prevTimeStamp = hmc.timeOfOldestEdit(); + + hmc.snapshot(0); + MemStoreSnapshot snapshot = hmc.snapshot(0); + if (useForce) { + // Make some assertions about what just happened. + assertTrue("History size has not increased", oldHistorySize < snapshot.getSize()); + long t = hmc.timeOfOldestEdit(); + assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE); + hmc.clearSnapshot(snapshot.getId()); + } else { + long t = hmc.timeOfOldestEdit(); + assertTrue("Time of oldest edit didn't remain the same", t == prevTimeStamp); + } + return prevTimeStamp; + } + + private void isExpectedRowWithoutTimestamps(final int rowIndex, List kvs) { + int i = 0; + for (Cell kv : kvs) { + byte[] expectedColname = makeQualifier(rowIndex, i++); + assertTrue("Column name", CellUtil.matchingQualifier(kv, expectedColname)); + // Value is column name as bytes. Usually result is + // 100 bytes in size at least. This is the default size + // for BytesWriteable. For comparison, convert bytes to + // String and trim to remove trailing null bytes. + assertTrue("Content", CellUtil.matchingValue(kv, expectedColname)); + } + } + + @Test public void testPuttingBackChunksAfterFlushing() throws IOException { + byte[] row = Bytes.toBytes("testrow"); + byte[] fam = Bytes.toBytes("testfamily"); + byte[] qf1 = Bytes.toBytes("testqualifier1"); + byte[] qf2 = Bytes.toBytes("testqualifier2"); + byte[] qf3 = Bytes.toBytes("testqualifier3"); + byte[] qf4 = Bytes.toBytes("testqualifier4"); + byte[] qf5 = Bytes.toBytes("testqualifier5"); + byte[] val = Bytes.toBytes("testval"); + + // Setting up memstore + memstore.add(new KeyValue(row, fam, qf1, val)); + memstore.add(new KeyValue(row, fam, qf2, val)); + memstore.add(new KeyValue(row, fam, qf3, val)); + + // Creating a snapshot + MemStoreSnapshot snapshot = memstore.snapshot(0); + assertEquals(3, memstore.getSnapshot().getCellsCount()); + + // Adding value to "new" memstore + assertEquals(0, memstore.getActive().getCellsCount()); + memstore.add(new KeyValue(row, fam, qf4, val)); + memstore.add(new KeyValue(row, fam, qf5, val)); + assertEquals(2, memstore.getActive().getCellsCount()); + memstore.clearSnapshot(snapshot.getId()); + + int chunkCount = chunkPool.getPoolSize(); + assertTrue(chunkCount > 0); + + } + + @Test public void testPuttingBackChunksWithOpeningScanner() throws IOException { + byte[] row = Bytes.toBytes("testrow"); + byte[] fam = Bytes.toBytes("testfamily"); + byte[] qf1 = Bytes.toBytes("testqualifier1"); + byte[] qf2 = Bytes.toBytes("testqualifier2"); + byte[] qf3 = Bytes.toBytes("testqualifier3"); + byte[] qf4 = Bytes.toBytes("testqualifier4"); + byte[] qf5 = Bytes.toBytes("testqualifier5"); + byte[] qf6 = Bytes.toBytes("testqualifier6"); + byte[] qf7 = Bytes.toBytes("testqualifier7"); + byte[] val = Bytes.toBytes("testval"); + + // Setting up memstore + memstore.add(new KeyValue(row, fam, qf1, val)); + memstore.add(new KeyValue(row, fam, qf2, val)); + memstore.add(new KeyValue(row, fam, qf3, val)); + + // Creating a snapshot + MemStoreSnapshot snapshot = memstore.snapshot(0); + assertEquals(3, memstore.getSnapshot().getCellsCount()); + + // Adding value to "new" memstore + assertEquals(0, memstore.getActive().getCellsCount()); + memstore.add(new KeyValue(row, fam, qf4, val)); + memstore.add(new KeyValue(row, fam, qf5, val)); + assertEquals(2, memstore.getActive().getCellsCount()); + + // opening scanner before clear the snapshot + List scanners = memstore.getScanners(0); + // Shouldn't putting back the chunks to pool,since some scanners are opening + // based on their data + memstore.clearSnapshot(snapshot.getId()); + + assertTrue(chunkPool.getPoolSize() == 0); + + // Chunks will be put back to pool after close scanners; + for (KeyValueScanner scanner : scanners) { + scanner.close(); + } + assertTrue(chunkPool.getPoolSize() > 0); + + // clear chunks + chunkPool.clearChunks(); + + // Creating another snapshot + + snapshot = memstore.snapshot(0); + // Adding more value + memstore.add(new KeyValue(row, fam, qf6, val)); + memstore.add(new KeyValue(row, fam, qf7, val)); + // opening scanners + scanners = memstore.getScanners(0); + // close scanners before clear the snapshot + for (KeyValueScanner scanner : scanners) { + scanner.close(); + } + // Since no opening scanner, the chunks of snapshot should be put back to + // pool + memstore.clearSnapshot(snapshot.getId()); + assertTrue(chunkPool.getPoolSize() > 0); + } + + @Test public void testPuttingBackChunksWithOpeningPipelineScanner() throws IOException { + byte[] row = Bytes.toBytes("testrow"); + byte[] fam = Bytes.toBytes("testfamily"); + byte[] qf1 = Bytes.toBytes("testqualifier1"); + byte[] qf2 = Bytes.toBytes("testqualifier2"); + byte[] qf3 = Bytes.toBytes("testqualifier3"); + byte[] val = Bytes.toBytes("testval"); + + // Setting up memstore + memstore.add(new KeyValue(row, fam, qf1, 1, val)); + memstore.add(new KeyValue(row, fam, qf2, 1, val)); + memstore.add(new KeyValue(row, fam, qf3, 1, val)); + + // Creating a pipeline + ((CompactingMemStore) memstore).disableCompaction(); + ((CompactingMemStore) memstore).flushInMemory(); + + // Adding value to "new" memstore + assertEquals(0, memstore.getActive().getCellsCount()); + memstore.add(new KeyValue(row, fam, qf1, 2, val)); + memstore.add(new KeyValue(row, fam, qf2, 2, val)); + assertEquals(2, memstore.getActive().getCellsCount()); + + // pipeline bucket 2 + ((CompactingMemStore) memstore).flushInMemory(); + // opening scanner before force flushing + List scanners = memstore.getScanners(0); + // Shouldn't putting back the chunks to pool,since some scanners are opening + // based on their data + ((CompactingMemStore) memstore).enableCompaction(); + // trigger compaction + ((CompactingMemStore) memstore).flushInMemory(); + + // Adding value to "new" memstore + assertEquals(0, memstore.getActive().getCellsCount()); + memstore.add(new KeyValue(row, fam, qf3, 3, val)); + memstore.add(new KeyValue(row, fam, qf2, 3, val)); + memstore.add(new KeyValue(row, fam, qf1, 3, val)); + assertEquals(3, memstore.getActive().getCellsCount()); + + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + } + + assertTrue(chunkPool.getPoolSize() == 0); + + // Chunks will be put back to pool after close scanners; + for (KeyValueScanner scanner : scanners) { + scanner.close(); + } + assertTrue(chunkPool.getPoolSize() > 0); + + // clear chunks + chunkPool.clearChunks(); + + // Creating another snapshot + + MemStoreSnapshot snapshot = memstore.snapshot(0); + memstore.clearSnapshot(snapshot.getId()); + + snapshot = memstore.snapshot(0); + // Adding more value + memstore.add(new KeyValue(row, fam, qf2, 4, val)); + memstore.add(new KeyValue(row, fam, qf3, 4, val)); + // opening scanners + scanners = memstore.getScanners(0); + // close scanners before clear the snapshot + for (KeyValueScanner scanner : scanners) { + scanner.close(); + } + // Since no opening scanner, the chunks of snapshot should be put back to + // pool + memstore.clearSnapshot(snapshot.getId()); + assertTrue(chunkPool.getPoolSize() > 0); + } + + ////////////////////////////////////////////////////////////////////////////// + // Compaction tests + ////////////////////////////////////////////////////////////////////////////// + public void testCompaction1Bucket() throws IOException { + int counter = 0; + String[] keys1 = { "A", "A", "B", "C" }; //A1, A2, B3, C4 + + // test 1 bucket + addRowsByKeys(memstore, keys1); + assertEquals(704, regionServicesForStores.getGlobalMemstoreTotalSize()); + assertEquals(4, memstore.getActive().getCellsCount()); + long size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(344, regionServicesForStores.getGlobalMemstoreTotalSize()); + for ( Segment s : memstore.getListOfSegments()) { + counter += s.getCellsCount(); + } + assertEquals(3, counter); + size = memstore.getFlushableSize(); + MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot + region.addAndGetGlobalMemstoreSize(-size); // simulate flusher + ImmutableSegment s = memstore.getSnapshot(); + assertEquals(3, s.getCellsCount()); + assertEquals(0, regionServicesForStores.getGlobalMemstoreTotalSize()); + + memstore.clearSnapshot(snapshot.getId()); + } + + public void testCompaction2Buckets() throws IOException { + + String[] keys1 = { "A", "A", "B", "C" }; + String[] keys2 = { "A", "B", "D" }; + + addRowsByKeys(memstore, keys1); + assertEquals(704, regionServicesForStores.getGlobalMemstoreTotalSize()); + long size = memstore.getFlushableSize(); + +// assertTrue( +// "\n\n<<< This is the active size with 4 keys - " + memstore.getActive().getSize() +// + ". This is the memstore flushable size - " + size + "\n",false); + + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(1000); + } + int counter = 0; + for ( Segment s : memstore.getListOfSegments()) { + counter += s.getCellsCount(); + } + assertEquals(3,counter); + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(344, regionServicesForStores.getGlobalMemstoreTotalSize()); + + addRowsByKeys(memstore, keys2); + assertEquals(872, regionServicesForStores.getGlobalMemstoreTotalSize()); + + size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + int i = 0; + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + if (i > 10000000) { + ((CompactingMemStore) memstore).debug(); + assertTrue("\n\n<<< Infinite loop! :( \n", false); + } + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + counter = 0; + for ( Segment s : memstore.getListOfSegments()) { + counter += s.getCellsCount(); + } + assertEquals(4,counter); + assertEquals(480, regionServicesForStores.getGlobalMemstoreTotalSize()); + + size = memstore.getFlushableSize(); + MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot + region.addAndGetGlobalMemstoreSize(-size); // simulate flusher + ImmutableSegment s = memstore.getSnapshot(); + assertEquals(4, s.getCellsCount()); + assertEquals(0, regionServicesForStores.getGlobalMemstoreTotalSize()); + + memstore.clearSnapshot(snapshot.getId()); + } + + public void testCompaction3Buckets() throws IOException { + + String[] keys1 = { "A", "A", "B", "C" }; + String[] keys2 = { "A", "B", "D" }; + String[] keys3 = { "D", "B", "B" }; + + addRowsByKeys(memstore, keys1); + assertEquals(704, region.getMemstoreSize()); + + long size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + + String tstStr = "\n\nFlushable size after first flush in memory:" + size + ". Is MemmStore in compaction?:" + + ((CompactingMemStore) memstore).isMemStoreFlushingInMemory(); + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(344, regionServicesForStores.getGlobalMemstoreTotalSize()); + + addRowsByKeys(memstore, keys2); + + tstStr += " After adding second part of the keys. Memstore size: " + + region.getMemstoreSize() + ", Memstore Total Size: " + + regionServicesForStores.getGlobalMemstoreTotalSize() + "\n\n"; + + assertEquals(872, regionServicesForStores.getGlobalMemstoreTotalSize()); + + ((CompactingMemStore) memstore).disableCompaction(); + size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline without compaction + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(872, regionServicesForStores.getGlobalMemstoreTotalSize()); + + addRowsByKeys(memstore, keys3); + assertEquals(1400, regionServicesForStores.getGlobalMemstoreTotalSize()); + + ((CompactingMemStore) memstore).enableCompaction(); + size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(480, regionServicesForStores.getGlobalMemstoreTotalSize()); + + size = memstore.getFlushableSize(); + MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot + region.addAndGetGlobalMemstoreSize(-size); // simulate flusher + ImmutableSegment s = memstore.getSnapshot(); + assertEquals(4, s.getCellsCount()); + assertEquals(0, regionServicesForStores.getGlobalMemstoreTotalSize()); + + memstore.clearSnapshot(snapshot.getId()); + + //assertTrue(tstStr, false); + } + + private void addRowsByKeys(final AbstractMemStore hmc, String[] keys) { + byte[] fam = Bytes.toBytes("testfamily"); + byte[] qf = Bytes.toBytes("testqualifier"); + for (int i = 0; i < keys.length; i++) { + long timestamp = System.currentTimeMillis(); + Threads.sleep(1); // to make sure each kv gets a different ts + byte[] row = Bytes.toBytes(keys[i]); + byte[] val = Bytes.toBytes(keys[i] + i); + KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); + hmc.add(kv); + LOG.debug("added kv: " + kv.getKeyString() + ", timestamp" + kv.getTimestamp()); + long size = AbstractMemStore.heapSizeChange(kv, true); + regionServicesForStores.addAndGetGlobalMemstoreSize(size); + } + } + + private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge { + long t = 1234; + + @Override public long currentTime() { + return t; + } + + public void setCurrentTimeMillis(long t) { + this.t = t; + } + } + +} From 4dc84b2f9a6f8653095d6a5554dbf3a2cc4dd86a Mon Sep 17 00:00:00 2001 From: anastas Date: Wed, 6 Apr 2016 16:35:15 +0300 Subject: [PATCH 4/9] Last Changes Before Review --- .../hadoop/hbase/regionserver/CompactingMemStore.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index c7873becf1f0..643ff8fdf8e6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -58,7 +58,7 @@ public class CompactingMemStore extends AbstractMemStore { public final static long DEEP_OVERHEAD_PER_PIPELINE_FLAT_ARRAY_ITEM = ClassSize.align( ClassSize.TIMERANGE_TRACKER + ClassSize.CELL_SKIPLIST_SET + ClassSize.CELL_ARRAY_MAP); - public final static double COPY_COMPACTION_THRESHOLD_FACTOR = 0.9; + public final static double IN_MEMORY_FLUSH_THRESHOLD_FACTOR = 0.9; public final static double COMPACTION_TRIGGER_REMAIN_FACTOR = 1; public final static boolean COMPACTION_PRE_CHECK = false; @@ -301,7 +301,7 @@ void flushInMemory() throws IOException { inMemoryFlushInProgress.set(true); // Speculative compaction execution, may be interrupted if flush is forced while // compaction is in progress - compactor.startCompact(0); + compactor.startCompact(); } } catch (IOException e) { LOG.warn("Unable to run memstore compaction. region " @@ -324,7 +324,7 @@ private ExecutorService getPool() { } private boolean shouldFlushInMemory() { - if(getActive().getSize() > COPY_COMPACTION_THRESHOLD_FACTOR *flushSizeLowerBound) { + if(getActive().getSize() > IN_MEMORY_FLUSH_THRESHOLD_FACTOR *flushSizeLowerBound) { // size above flush threshold return (allowCompaction.get() && !inMemoryFlushInProgress.get()); } @@ -415,7 +415,7 @@ private class MemStoreCompactor { * * is already an ongoing compaction (or pipeline is empty). */ - public boolean startCompact(int i) throws IOException { + public boolean startCompact() throws IOException { if (pipeline.isEmpty()) return false; // no compaction on empty pipeline // get the list of segments from the pipeline, the list is marked with specific version @@ -634,7 +634,7 @@ Cell getNextRow(final Cell cell) { // debug method, also for testing public void debug() { String msg = "active size="+getActive().getSize(); - msg += " threshold="+ COPY_COMPACTION_THRESHOLD_FACTOR *flushSizeLowerBound; + msg += " threshold="+ IN_MEMORY_FLUSH_THRESHOLD_FACTOR *flushSizeLowerBound; msg += " allow compaction is "+ (allowCompaction.get() ? "true" : "false"); msg += " inMemoryFlushInProgress is "+ (inMemoryFlushInProgress.get() ? "true" : "false"); LOG.debug(msg); From 834105fb3ac00d2e80dd973e4ade26af1dec8cb0 Mon Sep 17 00:00:00 2001 From: anastas Date: Wed, 6 Apr 2016 16:38:32 +0300 Subject: [PATCH 5/9] Very Last Changes Before Review --- .../apache/hadoop/hbase/regionserver/CompactingMemStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index 643ff8fdf8e6..32f7f4c550aa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -424,7 +424,7 @@ public boolean startCompact() throws IOException { LOG.info( "Starting the MemStore in-memory compaction for store " + store.getColumnFamilyName()); - doCompact(i); + doCompact(); return true; } @@ -452,7 +452,7 @@ private void releaseResources() { * The solo (per compactor) thread only reads the compaction pipeline. * There is at most one thread per memstore instance. */ - private void doCompact(int y) { + private void doCompact() { int cellsAfterComp = versionedList.getNumOfCells(); try { // the compaction processing // Phase I: estimate the compaction expedience - CHECK COMPACTION From d2a9a20b70c96ecd20770aa9fff11c2b29b7b7da Mon Sep 17 00:00:00 2001 From: anastas Date: Wed, 13 Apr 2016 10:14:25 +0300 Subject: [PATCH 6/9] After the Code Review --- .../apache/hadoop/hbase/util/ClassSize.java | 22 +- .../hbase/regionserver/AbstractMemStore.java | 4 +- .../hbase/regionserver/CellArrayMap.java | 2 +- .../hbase/regionserver/CellChunkMap.java | 19 +- .../hbase/regionserver/CellFlatMap.java | 111 +++++----- .../regionserver/CompactingMemStore.java | 193 ++++++----------- .../regionserver/CompactionPipeline.java | 45 ++-- .../hbase/regionserver/HeapMemStoreLAB.java | 10 +- .../hbase/regionserver/ImmutableSegment.java | 205 ++++++++++++------ .../hbase/regionserver/MemStoreChunkPool.java | 3 +- .../MemStoreCompactorIterator.java | 24 +- .../hadoop/hbase/regionserver/Segment.java | 21 +- .../hbase/regionserver/SegmentFactory.java | 31 +-- .../regionserver/VersionedSegmentsList.java | 9 +- .../apache/hadoop/hbase/io/TestHeapSize.java | 2 +- 15 files changed, 367 insertions(+), 334 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java index 1c1b2c190f98..b0f2e9c83df4 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java @@ -87,8 +87,14 @@ public class ClassSize { /** Overhead for CellArrayMap */ public static final int CELL_ARRAY_MAP; + /** Overhead for CellArrayMap */ + public static final int CELL_CHUNK_MAP; + /** Overhead for Cell Array Entry */ - public static final int CELL_ARRAY_ENTRY; + public static final int CELL_ARRAY_MAP_ENTRY; + + /** Overhead for CellChunkMap Entry */ + public static final int CELL_CHUNK_MAP_ENTRY; /** Overhead for ReentrantReadWriteLock */ public static final int REENTRANT_LOCK; @@ -115,7 +121,7 @@ public class ClassSize { public static final int TIMERANGE_TRACKER; /** Overhead for CellSkipListSet */ - public static final int CELL_SKIPLIST_SET; + public static final int CELL_SET; public static final int STORE_SERVICES; @@ -181,13 +187,19 @@ public class ClassSize { // The size changes from jdk7 to jdk8, estimate the size rather than use a conditional CONCURRENT_SKIPLISTMAP = (int) estimateBase(ConcurrentSkipListMap.class, false); - CELL_ARRAY_MAP = align(OBJECT); + CELL_ARRAY_MAP = align(2*OBJECT + Bytes.SIZEOF_LONG + Bytes.SIZEOF_BOOLEAN + + 2*Bytes.SIZEOF_INT + REFERENCE); + + CELL_CHUNK_MAP = align(2*OBJECT + Bytes.SIZEOF_LONG + Bytes.SIZEOF_BOOLEAN + + 4*Bytes.SIZEOF_INT + 2*REFERENCE); CONCURRENT_SKIPLISTMAP_ENTRY = align( align(OBJECT + (3 * REFERENCE)) + /* one node per entry */ align((OBJECT + (3 * REFERENCE))/2)); /* one index per two entries */ - CELL_ARRAY_ENTRY = align(2*REFERENCE + 2*Bytes.SIZEOF_INT); + CELL_ARRAY_MAP_ENTRY = align(OBJECT + 2*REFERENCE + 2*Bytes.SIZEOF_INT); + + CELL_CHUNK_MAP_ENTRY = align(3*Bytes.SIZEOF_INT); REENTRANT_LOCK = align(OBJECT + (3 * REFERENCE)); @@ -205,7 +217,7 @@ public class ClassSize { TIMERANGE_TRACKER = align(ClassSize.OBJECT + Bytes.SIZEOF_LONG * 2); - CELL_SKIPLIST_SET = align(OBJECT + REFERENCE); + CELL_SET = align(OBJECT + REFERENCE); STORE_SERVICES = align(OBJECT + REFERENCE + ATOMIC_LONG); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java index 767551dff423..70b07fa214f0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java @@ -65,14 +65,14 @@ public abstract class AbstractMemStore implements MemStore { public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD + 2 * (ClassSize.ATOMIC_LONG + ClassSize.TIMERANGE_TRACKER + - ClassSize.CELL_SKIPLIST_SET + ClassSize.CONCURRENT_SKIPLISTMAP)); + ClassSize.CELL_SET + ClassSize.CONCURRENT_SKIPLISTMAP)); protected AbstractMemStore(final Configuration conf, final CellComparator c) { this.conf = conf; this.comparator = c; resetCellSet(); - this.snapshot = SegmentFactory.instance().createImmutableSegment(conf, c, 0); + this.snapshot = SegmentFactory.instance().createImmutableSegment(c, 0); this.snapshotId = NO_SNAPSHOT_ID; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java index 8508abeb5a23..8cc8b206057a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java @@ -46,7 +46,7 @@ protected CellFlatMap createCellFlatMap(Comparator comparator, int } @Override - protected Cell getCellFromIndex(int i) { + protected Cell getCell(int i) { return block[i]; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java index 2ed5a0dcd263..ec53e75aa6b8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java @@ -33,7 +33,7 @@ * and length in bytes in B (integer). In order to save reference to byte array we use the Chunk's * indexes given by MSLAB (also integer). * - * The B memory layout: + * The CellChunkMap memory layout relevant to a deeper byte array B: * * <----------------- first Cell ---------------------> <-------------- second Cell --- ... * ------------------------------------------------------------------------------------- ... @@ -43,12 +43,23 @@ * ------------------------------------------------------------------------------------- ... */ public class CellChunkMap extends CellFlatMap { - + // TODO: once Chunk class is out of HeapMemStoreLAB class we are going to use MemStoreLAB and + // not HeapMemStoreLAB private final HeapMemStoreLAB.Chunk[] chunks; private final HeapMemStoreLAB memStoreLAB; - private int numOfCellsInsideChunk; + private final int numOfCellsInsideChunk; public static final int BYTES_IN_CELL = 3*(Integer.SIZE / Byte.SIZE); // each Cell requires 3 integers + /* C-tor for increasing map starting from index zero */ + /* The given Cell array on given Chunk array must be ordered. */ + public CellChunkMap(Comparator comparator, HeapMemStoreLAB memStoreLAB, + HeapMemStoreLAB.Chunk[] chunks, int max, int chunkSize) { + super(comparator,0,max,false); + this.chunks = chunks; + this.memStoreLAB = memStoreLAB; + this.numOfCellsInsideChunk = chunkSize / BYTES_IN_CELL; + } + /* The given Cell array on given Chunk array must be ordered. */ public CellChunkMap(Comparator comparator, HeapMemStoreLAB memStoreLAB, HeapMemStoreLAB.Chunk[] chunks, int min, int max, int chunkSize, boolean d) { @@ -67,7 +78,7 @@ protected CellFlatMap createCellFlatMap(Comparator comparator, int } @Override - protected Cell getCellFromIndex(int i) { + protected Cell getCell(int i) { // find correct chunk int chunkIndex = (i / numOfCellsInsideChunk); byte[] block = chunks[chunkIndex].getData(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java index ce7b74f083a7..a209abaeedf5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java @@ -60,7 +60,7 @@ protected abstract CellFlatMap createCellFlatMap(Comparator compar int max, boolean descending); /* Returns the i-th cell in the cell block */ - protected abstract Cell getCellFromIndex(int i); + protected abstract Cell getCell(int i); /** * Binary search for a given key in between given boundaries of the array. @@ -77,11 +77,12 @@ private int find(Cell needle) { while (begin <= end) { int mid = begin + ((end - begin) / 2); - Cell midCell = getCellFromIndex(mid); + Cell midCell = getCell(mid); int compareRes = comparator.compare(midCell, needle); - if (compareRes == 0) return mid; // 0 means equals. We found the key - else if (compareRes < 0) { + if (compareRes == 0) { + return mid; // 0 means equals. We found the key + } else if (compareRes < 0) { // midCell is less than needle so we need to look at farther up begin = mid + 1; } else { @@ -97,8 +98,9 @@ else if (compareRes < 0) { ** the key should be inclusive or exclusive */ private int getValidIndex(Cell key, boolean inclusive) { int index = find(key); - if (inclusive && index >= 0) - index = (descending) ? index-1 : index+1; + if (inclusive && index >= 0) { + index = (descending) ? index - 1 : index + 1; + } return Math.abs(index); } @@ -114,7 +116,7 @@ public int size() { @Override public boolean isEmpty() { - return (maxCellIdx==minCellIdx); + return (size()==0); } @@ -127,7 +129,9 @@ public ConcurrentNavigableMap subMap( Cell fromKey, int toIndex = getValidIndex(toKey, toInclusive); int fromIndex = (getValidIndex(fromKey, !fromInclusive)); - if (fromIndex > toIndex) throw new IllegalArgumentException("inconsistent range"); + if (fromIndex > toIndex) { + throw new IllegalArgumentException("inconsistent range"); + } return createCellFlatMap(comparator, fromIndex, toIndex, descending); } @@ -167,68 +171,59 @@ public ConcurrentNavigableMap tailMap(Cell k) { // -------------------------------- Key's getters -------------------------------- @Override public Cell firstKey() { - if (isEmpty()) return null; - if (descending) getCellFromIndex(maxCellIdx-1); - return getCellFromIndex(minCellIdx); + if (isEmpty()) { + return null; + } + return descending ? getCell(maxCellIdx - 1) : getCell(minCellIdx); } @Override public Cell lastKey() { - if (isEmpty()) return null; - if (descending) return getCellFromIndex(minCellIdx); - return getCellFromIndex(maxCellIdx-1); + if (isEmpty()) { + return null; + } + return descending ? getCell(minCellIdx) : getCell(maxCellIdx - 1); } @Override public Cell lowerKey(Cell k) { - if (isEmpty()) return null; - int index = find(k); - if (descending) { - if (index >= 0) index++; // There's a key exactly equal. - else index = -(index + 1); - } else { - if (index >= 0) index--; // There's a key exactly equal. - else index = -(index + 1) - 1; + if (isEmpty()) { + return null; } - return (index < minCellIdx || index >= maxCellIdx) ? null : getCellFromIndex(index); + int index = find(k); + // If index>=0 there's a key exactly equal + index = (index>=0) ? index-1 : -(index); + return (index < minCellIdx || index >= maxCellIdx) ? null : getCell(index); } @Override public Cell floorKey(Cell k) { - if (isEmpty()) return null; - int index = find(k); - if (descending) { - if (index < 0) index = -(index + 1); - } else { - if (index < 0) index = -(index + 1) - 1; + if (isEmpty()) { + return null; } - return (index < minCellIdx || index >= maxCellIdx) ? null : getCellFromIndex(index); + int index = find(k); + index = (index>=0) ? index : -(index); + return (index < minCellIdx || index >= maxCellIdx) ? null : getCell(index); } @Override public Cell ceilingKey(Cell k) { - if (isEmpty()) return null; - int index = find(k); - if (descending) { - if (index < 0) index = -(index + 1) - 1; - } else { - if (index < 0) index = -(index + 1); + if (isEmpty()) { + return null; } - return (index < minCellIdx || index >= maxCellIdx) ? null : getCellFromIndex(index); + int index = find(k); + index = (index>=0) ? index : -(index)+1; + return (index < minCellIdx || index >= maxCellIdx) ? null : getCell(index); } @Override public Cell higherKey(Cell k) { - if (isEmpty()) return null; - int index = find(k); - if (descending) { - if (index >= 0) index--; // There's a key exactly equal. - else index = -(index + 1) - 1; - } else { - if (index >= 0) index++; // There's a key exactly equal. - else index = -(index + 1); + if (isEmpty()) { + return null; } - return (index < minCellIdx || index >= maxCellIdx) ? null : getCellFromIndex(index); + int index = find(k); + index = (index>=0) ? index+1 : -(index)+1; + return (index < minCellIdx || index >= maxCellIdx) ? null : getCell(index); } @Override @@ -245,10 +240,7 @@ public boolean containsValue(Object o) { // use containsKey(Object o) instead @Override public Cell get(Object o) { int index = find((Cell) o); - if (index >= 0) { - return getCellFromIndex(index); - } - return null; + return (index >= 0) ? getCell(index) : null; } // -------------------------------- Entry's getters -------------------------------- @@ -358,7 +350,7 @@ public NavigableSet keySet() { @Override public Collection values() { - return new CellBlocksCollection(); + return new CellFlatMapCollection(); } @Override @@ -368,10 +360,10 @@ public Set> entrySet() { // -------------------------------- Iterator K -------------------------------- - private final class CellBlocksIterator implements Iterator { + private final class CellFlatMapIterator implements Iterator { int index; - private CellBlocksIterator() { + private CellFlatMapIterator() { index = descending ? maxCellIdx-1 : minCellIdx; } @@ -382,9 +374,12 @@ public boolean hasNext() { @Override public Cell next() { - Cell result = getCellFromIndex(index); - if (descending) index--; - else index++; + Cell result = getCell(index); + if (descending) { + index--; + } else { + index++; + } return result; } @@ -395,7 +390,7 @@ public void remove() { } // -------------------------------- Collection -------------------------------- - private final class CellBlocksCollection implements Collection { + private final class CellFlatMapCollection implements Collection { @Override public int size() { @@ -419,7 +414,7 @@ public boolean contains(Object o) { @Override public Iterator iterator() { - return new CellBlocksIterator(); + return new CellFlatMapIterator(); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index 32f7f4c550aa..799e31fc0834 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -19,7 +19,9 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; -import java.util.*; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; @@ -32,8 +34,6 @@ import org.apache.hadoop.hbase.executor.EventHandler; import org.apache.hadoop.hbase.executor.EventType; import org.apache.hadoop.hbase.executor.ExecutorService; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -54,14 +54,17 @@ public class CompactingMemStore extends AbstractMemStore { public final static long DEEP_OVERHEAD_PER_PIPELINE_ITEM = ClassSize.align( ClassSize.TIMERANGE_TRACKER + - ClassSize.CELL_SKIPLIST_SET + ClassSize.CONCURRENT_SKIPLISTMAP); + ClassSize.CELL_SET + ClassSize.CONCURRENT_SKIPLISTMAP); public final static long DEEP_OVERHEAD_PER_PIPELINE_FLAT_ARRAY_ITEM = ClassSize.align( ClassSize.TIMERANGE_TRACKER + - ClassSize.CELL_SKIPLIST_SET + ClassSize.CELL_ARRAY_MAP); + ClassSize.CELL_SET + ClassSize.CELL_ARRAY_MAP); public final static double IN_MEMORY_FLUSH_THRESHOLD_FACTOR = 0.9; public final static double COMPACTION_TRIGGER_REMAIN_FACTOR = 1; public final static boolean COMPACTION_PRE_CHECK = false; + static final String COMPACTING_MEMSTORE_TYPE_KEY = "hbase.hregion.compacting.memstore.type"; + static final int COMPACTING_MEMSTORE_TYPE_DEFAULT = 1; + private static final Log LOG = LogFactory.getLog(CompactingMemStore.class); private HStore store; private RegionServicesForStores regionServices; @@ -79,7 +82,7 @@ public class CompactingMemStore extends AbstractMemStore { public enum Type { COMPACT_TO_SKIPLIST_MAP, COMPACT_TO_ARRAY_MAP, - COMPACT_TO_CHUNK_MAP + COMPACT_TO_CHUNK_MAP; } private Type type = Type.COMPACT_TO_SKIPLIST_MAP; @@ -92,17 +95,22 @@ public CompactingMemStore(Configuration conf, CellComparator c, this.pipeline = new CompactionPipeline(getRegionServices()); this.compactor = new MemStoreCompactor(); initFlushSizeLowerBound(conf); + int t = conf.getInt(COMPACTING_MEMSTORE_TYPE_KEY, COMPACTING_MEMSTORE_TYPE_DEFAULT); + switch (t) { + case 1: type = Type.COMPACT_TO_SKIPLIST_MAP; + break; + case 2: type = Type.COMPACT_TO_ARRAY_MAP; + break; + case 3: type = Type.COMPACT_TO_CHUNK_MAP; + break; + } } + // C-tor for testing public CompactingMemStore(Configuration conf, CellComparator c, HStore store, RegionServicesForStores regionServices, Type type) throws IOException { - super(conf, c); - this.store = store; - this.regionServices = regionServices; - this.pipeline = new CompactionPipeline(getRegionServices()); - this.compactor = new MemStoreCompactor(); + this(conf,c,store,regionServices); this.type = type; - initFlushSizeLowerBound(conf); } private void initFlushSizeLowerBound(Configuration conf) { @@ -398,32 +406,26 @@ public void process() throws IOException { */ private class MemStoreCompactor { - // a static version of the segment list from the pipeline + // a snapshot of the compaction pipeline segment list private VersionedSegmentsList versionedList; - // a flag raised when compaction is requested to stop private final AtomicBoolean isInterrupted = new AtomicBoolean(false); - - // the limit to the size of the groups to be later returned by compactingScanner + // the limit to the size of the groups to be later provided to MemStoreCompactorIterator private final int compactionKVMax = getConfiguration().getInt( - HConstants.COMPACTION_KV_MAX, - HConstants.COMPACTION_KV_MAX_DEFAULT); + HConstants.COMPACTION_KV_MAX, HConstants.COMPACTION_KV_MAX_DEFAULT); /** ---------------------------------------------------------------------- * The request to dispatch the compaction asynchronous task. * The method returns true if compaction was successfully dispatched, or false if there - * * is already an ongoing compaction (or pipeline is empty). */ public boolean startCompact() throws IOException { if (pipeline.isEmpty()) return false; // no compaction on empty pipeline - - // get the list of segments from the pipeline, the list is marked with specific version + // get a snapshot of the list of the segments from the pipeline, + // this local copy of the list is marked with specific version versionedList = pipeline.getVersionedList(); - LOG.info( "Starting the MemStore in-memory compaction for store " + store.getColumnFamilyName()); - doCompact(); return true; } @@ -454,8 +456,8 @@ private void releaseResources() { */ private void doCompact() { int cellsAfterComp = versionedList.getNumOfCells(); - try { // the compaction processing - // Phase I: estimate the compaction expedience - CHECK COMPACTION + try { + // Phase I (optional): estimate the compaction expedience - CHECK COMPACTION if (COMPACTION_PRE_CHECK) { cellsAfterComp = countCellsForCompaction(); @@ -464,30 +466,15 @@ private void doCompact() { // too much cells "survive" the possible compaction we do not want to compact! LOG.debug("Stopping the unworthy MemStore in-memory compaction for store " + getFamilyName()); - // Looking for Segment in pipeline with SkipList index, to make it flat - ImmutableSegment segment = versionedList.getSkipListSegment(); - if (segment == null) { - Thread.currentThread().interrupt(); - return; - } - // Flatten the given segment with SkipList index and replace it in the pipeline - flattenSegment(segment, cellsAfterComp); + // Looking for Segment in the pipeline with SkipList index, to make it flat + pipeline.flattenOneSegment(versionedList.getVersion()); return; } } + // Phase II: create the new compacted ImmutableSegment - START COMPACTION ImmutableSegment result = null; - // Phase II: create the compacted ImmutableSegment - START COMPACTION if (!isInterrupted.get()) { - switch (type) { - case COMPACT_TO_SKIPLIST_MAP: result = compactToNotFlatSegment(cellsAfterComp); - break; - case COMPACT_TO_ARRAY_MAP: result = compactToFlatSegment(cellsAfterComp, true); - break; - case COMPACT_TO_CHUNK_MAP: result = compactToFlatSegment(cellsAfterComp, false); - break; - default: // sanity check - org.junit.Assert.assertTrue("\n\n<<< Wrong compacting MemStore type", false); - } + result = compact(cellsAfterComp); } // Phase III: swap the old compaction pipeline - END COMPACTION if (!isInterrupted.get()) { @@ -502,98 +489,62 @@ private void doCompact() { releaseResources(); inMemoryFlushInProgress.set(false); } - - } - - /** - * Creates a new Segment with CellFlatMap from the given single old - * Segment based on ConcurrentSkipListMap - */ - private void flattenSegment(ImmutableSegment segment, int numOfCells) throws IOException { - - LOG.debug("Flattening Segment " + segment); - // Update the pipeline scanner (list of scanners) to scan only Skip List based Segment - List scanners = new ArrayList(); - scanners.add(segment.getSegmentScanner(store.getSmallestReadPoint())); - ImmutableSegment newSegment = createFlatFromSegmentWithSkipList(numOfCells, segment); - pipeline.replace(versionedList, segment, newSegment); - - return; } - /** - * COMPACT TO ARRAY MAP OR TO CHUNK MAP - * Updates the given single Segment using the internal store scanner, - * who in turn uses ScanQueryMatcher + /**---------------------------------------------------------------------- + * The compaction is the creation of the relevant ImmutableSegment based on + * the Compactor Itertor */ - private ImmutableSegment compactToFlatSegment(int numOfCells, boolean array) + private ImmutableSegment compact(int numOfCells) throws IOException { + ImmutableSegment result = null; MemStoreCompactorIterator iterator = - new MemStoreCompactorIterator( - numOfCells, versionedList.getStoreSegments(), getComparator(), compactionKVMax, store); - - ImmutableSegment result = SegmentFactory.instance() - .createImmutableSegment(getConfiguration(), getComparator(), numOfCells, iterator, array); - - iterator.remove(); - return result; - } - - /** - * COMPACT TO SKIPLIST MAP - */ - private ImmutableSegment compactToNotFlatSegment(int cells) throws IOException { - ImmutableSegment result = SegmentFactory.instance() // create the scanner - .createImmutableSegment(getConfiguration(), getComparator(), - CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_ITEM); - - MemStoreCompactorIterator iterator = - new MemStoreCompactorIterator( - cells,versionedList.getStoreSegments(),getComparator(),compactionKVMax, store); - - while (iterator.hasNext()) { - Cell c = iterator.next(); - // The scanner is doing all the elimination logic - // now we just copy it to the new segment - KeyValue kv = KeyValueUtil.ensureKeyValue(c); - Cell newKV = result.maybeCloneWithAllocator(kv); - result.internalAdd(newKV); + new MemStoreCompactorIterator(versionedList.getStoreSegments(), getComparator(), + compactionKVMax, store); + try { + switch (type) { + case COMPACT_TO_SKIPLIST_MAP: + result = SegmentFactory.instance() + .createImmutableSegment(getConfiguration(), getComparator(), iterator); + break; + case COMPACT_TO_ARRAY_MAP: + result = SegmentFactory.instance() + .createImmutableSegment( + getConfiguration(), getComparator(), iterator, numOfCells, true); + break; + case COMPACT_TO_CHUNK_MAP: + result = SegmentFactory.instance() + .createImmutableSegment( + getConfiguration(), getComparator(), iterator, numOfCells, false); + break; + default: throw new RuntimeException("Unknown type " + type); // sanity check + } + } finally { + iterator.close(); } - iterator.remove(); - return result; - } - - /** - * FLATTENING - * Create CellSet for flat representation - * The MSLAB Chunks with the real data remain the same - */ - private ImmutableSegment createFlatFromSegmentWithSkipList(int numOfCells, - ImmutableSegment segment) throws IOException { - LinkedList segmentList = new LinkedList(); - segmentList.add(segment); - - MemStoreCompactorIterator iterator = - new MemStoreCompactorIterator(4,segmentList,getComparator(),compactionKVMax, store); - - ImmutableSegment result = SegmentFactory.instance() - .createImmutableSegment(numOfCells, segment, iterator); - iterator.remove(); return result; } + /**---------------------------------------------------------------------- + * COUNT CELLS TO ESTIMATE THE EFFICIENCY OF THE FUTURE COMPACTION + */ private int countCellsForCompaction() throws IOException { - MemStoreCompactorIterator iterator = - new MemStoreCompactorIterator( - 3,versionedList.getStoreSegments(),getComparator(),compactionKVMax, store); int cnt = 0; - while (iterator.next() != null) { - cnt++; + MemStoreCompactorIterator iterator = + new MemStoreCompactorIterator(versionedList.getStoreSegments(),getComparator(), + compactionKVMax, store); + + try { + while (iterator.next() != null) { + cnt++; + } + } finally { + iterator.close(); } - iterator.remove(); + return cnt; } } // end of the MemStoreCompactor Class diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java index f75aeba1c813..fa51f79ad7f6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java @@ -98,7 +98,7 @@ public boolean swap(VersionedSegmentsList versionedList, ImmutableSegment segmen +"Just before the swap the number of segments in pipeline is:" +versionedList.getStoreSegments().size() +", and the number of cells in new segment is:"+segment.getCellsCount()); - swapSuffix(suffix,segment); + swapSuffix(suffix, segment); } if(region != null) { // update the global memstore size counter @@ -113,29 +113,35 @@ public boolean swap(VersionedSegmentsList versionedList, ImmutableSegment segmen } /** - * Replaces a single segment in the pipeline with the new flattened segment. - * Replacing only if there were no changes in the pipeline. This is achieved - * by requiring the version (from versionedList) being the same. - * @param versionedList tail of the pipeline that was taken for compaction and holds the version - * @param newSegment new compacted segment - * @return true iff replaced with new compacted segment + * If the caller holds the current version, go over the the pipeline and try to flatten each + * segment. Flattening is replacing the ConcurrentSkipListMap based CellSet to CellArrayMAp based. + * Flattening of the segment that initially is not based on ConcurrentSkipListMap has no effect. + * Return after one segment was successfully flatten. + * + * @return true iff a segment was successfully flattened */ - public boolean replace(VersionedSegmentsList versionedList, - ImmutableSegment oldSegment, ImmutableSegment newSegment) { + public boolean flattenOneSegment(long requesterVersion) { - if(versionedList.getVersion() != version) { + if(requesterVersion != version) { return false; } synchronized (pipeline){ - if(versionedList.getVersion() != version) { + if(requesterVersion != version) { return false; } - LOG.info("Replacing one pipeline segment with flattened segment."); - replaceSegment(oldSegment,newSegment); + + for (ImmutableSegment s : pipeline) { + if (s.flatten()) { + LOG.info("Compaction pipeline segment " + s + " was flattened."); + return true; + } + } + } - // do not update the global memstore size counter, because all the cells remain in place - return true; + // do not update the global memstore size counter and do not increase the version, + // because all the cells remain in place + return false; } public boolean isEmpty() { @@ -175,15 +181,6 @@ private void swapSuffix(LinkedList suffix, ImmutableSegment se pipeline.addLast(segment); } - /* Replacing one representation of the segment with another. Literally removing a segment and - adding a new one (with same keys). The order of the segments is not kept */ - private void replaceSegment(ImmutableSegment oldSegment, ImmutableSegment newSegment) { - version++; - // don't call oldSegment.close() because its MSLAB is transferred to the new segment - pipeline.remove(oldSegment); - pipeline.add(newSegment); - } - private ImmutableSegment removeLast() { version++; return pipeline.removeLast(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java index 00df16423050..63617251461d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.util.ByteRange; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.SimpleMutableByteRange; import com.google.common.base.Preconditions; @@ -102,6 +103,11 @@ public HeapMemStoreLAB(Configuration conf) { @Override public ByteRange allocateBytes(int size) { Preconditions.checkArgument(size >= 0, "negative size"); + return allocateBytesWithID(size).getFirst(); + } + + public Pair allocateBytesWithID(int size) { + Preconditions.checkArgument(size >= 0, "negative size"); // Callers should satisfy large allocations directly from JVM since they // don't cause fragmentation as badly. @@ -112,12 +118,12 @@ public ByteRange allocateBytes(int size) { while (true) { Chunk c = getOrMakeChunk(); - // Try to allocate from this chunk + // Try to allocate from this chunk int allocOffset = c.alloc(size); if (allocOffset != -1) { // We succeeded - this is the common case - small alloc // from a big buffer - return new SimpleMutableByteRange(c.data, allocOffset, size); + return new Pair<>(new SimpleMutableByteRange(c.data, allocOffset, size),c.getId()); } // not enough space! diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java index 64bebd58be2d..bfed35791e5c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java @@ -18,15 +18,21 @@ */ package org.apache.hadoop.hbase.regionserver; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.ClassSize; +import org.apache.hadoop.hbase.util.CollectionBackedScanner; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.SimpleMutableByteRange; import org.apache.hadoop.hbase.util.ByteRange; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.CollectionBackedScanner; + +import java.io.IOException; /** * ImmutableSegment is an abstract class that extends the API supported by a {@link Segment}, @@ -37,91 +43,166 @@ @InterfaceAudience.Private public class ImmutableSegment extends Segment { - private boolean isFlat; // whether it is based on CellFlatMap or ConcurrentSkipListMap + private boolean isFlat; // whether it is based on CellFlatMap or ConcurrentSkipListMap - /** - * Builds a special scanner for the MemStoreSnapshot object that is different than the - * general segment scanner. - * @return a special scanner for the MemStoreSnapshot object + ///////////////////// CONSTRUCTORS ///////////////////// + /**------------------------------------------------------------------------ + * C-tor to be used when new ImmutableSegment is being built from a Mutable one. + * This C-tor should be used when active MutableSegment is pushed into the compaction + * pipeline and becomes an ImmutableSegment. */ - public KeyValueScanner getKeyValueScanner() { - return new CollectionBackedScanner(getCellSet(), getComparator()); - } - - public boolean isFlat() { - return isFlat; - } - protected ImmutableSegment(Segment segment) { super(segment); isFlat = false; } - // C-tor by flattening other (old) segment - protected ImmutableSegment(ImmutableSegment oldSegment, MemStoreCompactorIterator iterator, - MemStoreLAB memStoreLAB, int numOfCells, long constantCellSizeOverhead) { - - super(null,oldSegment.getComparator(),memStoreLAB, - CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_FLAT_ARRAY_ITEM, constantCellSizeOverhead); - CellSet cs = this.createArrayBasedCellSet(numOfCells, iterator, false); // build the CellSet - this.setCellSet(cs); // update the CellSet of the new Segment - isFlat = true; - } - - // C-tor by compaction to CellArrayMap or CellChunkMap + /**------------------------------------------------------------------------ + * C-tor to be used when new ImmutableSegment is a result of compaction of a list + * of older ImmutableSegments. + * The given iterator returns the Cells that "survived" the compaction. + * According to the boolean parameter "array" the new ImmutableSegment is built based on + * CellArrayMap or CellChunkMap. + */ protected ImmutableSegment( final Configuration conf, CellComparator comparator, MemStoreCompactorIterator iterator, - MemStoreLAB memStoreLAB, int numOfCells, long constantCellSizeOverhead, boolean array) { + MemStoreLAB memStoreLAB, int numOfCells, boolean array) { super(null, comparator, memStoreLAB, - CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_FLAT_ARRAY_ITEM, constantCellSizeOverhead); + CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_FLAT_ARRAY_ITEM, + array ? ClassSize.CELL_ARRAY_MAP_ENTRY : ClassSize.CELL_CHUNK_MAP_ENTRY); CellSet cs = null; // build the CellSet Cell array or Byte array based if (array) { - cs = this.createArrayBasedCellSet(numOfCells, iterator, true); + cs = createCellArrayMapSet(numOfCells, iterator); } else { - cs = this.createChunkBasedCellSet(numOfCells, iterator, conf); + cs = createCellChunkMapSet(numOfCells, iterator, conf); } - this.setCellSet(cs); // update the CellSet of the new Segment + this.setCellSet(null, cs); // update the CellSet of the new Segment isFlat = true; } - private CellSet createArrayBasedCellSet( - int numOfCells, MemStoreCompactorIterator iterator, boolean allocateFromMSLAB) { + /**------------------------------------------------------------------------ + * C-tor to be used when new SKIP-LIST BASED ImmutableSegment is a result of compaction of a + * list of older ImmutableSegments. + * The given iterator returns the Cells that "survived" the compaction. + */ + protected ImmutableSegment( + CellComparator comparator, MemStoreCompactorIterator iterator, MemStoreLAB memStoreLAB) { + + super(null, comparator, memStoreLAB, CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_ITEM, + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); + + while (iterator.hasNext()) { + Cell c = iterator.next(); + // The scanner is doing all the elimination logic + // now we just copy it to the new segment + KeyValue kv = KeyValueUtil.ensureKeyValue(c); + Cell newKV = maybeCloneWithAllocator(kv); + internalAdd(newKV); + } + isFlat = false; + } + + ///////////////////// PUBLIC METHODS ///////////////////// + /** + * Builds a special scanner for the MemStoreSnapshot object that is different than the + * general segment scanner. + * @return a special scanner for the MemStoreSnapshot object + */ + public KeyValueScanner getKeyValueScanner() { + return new CollectionBackedScanner(getCellSet(), getComparator()); + } + + /**------------------------------------------------------------------------ + * Change the CellSet of this ImmutableSegment from one based on ConcurrentSkipListMap to one + * based on CellArrayMap. + * If this ImmutableSegment is not based on ConcurrentSkipListMap , this is NOP + * For now the change from ConcurrentSkipListMap to CellChunkMap is not supported, because + * this requires the Cell to know on which Chunk it is placed. + * + * Synchronization of the CellSet replacement: + * The reference to the CellSet is AtomicReference and is updated only when ImmutableSegment + * is constructed (single thread) or flattened. The flattening happens as part of a single + * thread of compaction, but to be on the safe side the initial CellSet is locally saved + * before the flattening and then replaced using CAS instruction. + */ + public boolean flatten() { + if (isFlat) return false; + CellSet oldCellSet = getCellSet(); + int numOfCells = getCellsCount(); + + // each Cell is now represented in either in CellArrayMap or in CellChunkMap + constantCellMetaDataSize = ClassSize.CELL_ARRAY_MAP_ENTRY; + + // arrange the meta-data size, decrease all meta-data sizes related to SkipList + incSize( + -(ClassSize.CONCURRENT_SKIPLISTMAP + numOfCells * ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY)); + // add size of CellArrayMap and meta-data overhead per Cell + incSize(ClassSize.CELL_CHUNK_MAP + numOfCells * constantCellMetaDataSize); + + CellSet newCellSet = recreateCellArrayMapSet(numOfCells); // build the CellSet CellArrayMap based + setCellSet(oldCellSet,newCellSet); + return true; + } + + ///////////////////// PRIVATE METHODS ///////////////////// + /*------------------------------------------------------------------------*/ + // Create CellSet based on CellArrayMap from compacting iterator + private CellSet createCellArrayMapSet(int numOfCells, MemStoreCompactorIterator iterator) { Cell[] cells = new Cell[numOfCells]; // build the Cell Array int i = 0; while (iterator.hasNext()) { Cell c = iterator.next(); - if (allocateFromMSLAB) { - // The scanner behind the iterator is doing all the elimination logic - // now we just copy it to the new segment (also MSLAB copy) - KeyValue kv = KeyValueUtil.ensureKeyValue(c); - Cell newKV = maybeCloneWithAllocator(kv); - cells[i++] = newKV; - // flattening = false, compaction case, counting both Heap and MetaData size - updateMetaInfo(c,true,!allocateFromMSLAB); - } else { - cells[i++] = c; - // flattening = true, flattening case, counting only MetaData size - updateMetaInfo(c,true,!allocateFromMSLAB); - } - + // The scanner behind the iterator is doing all the elimination logic + // now we just copy it to the new segment (also MSLAB copy) + KeyValue kv = KeyValueUtil.ensureKeyValue(c); + cells[i++] = maybeCloneWithAllocator(kv); + // last parameter false, because in compaction count both Heap (Data) and MetaData size + updateMetaInfo(c,true); } // build the immutable CellSet CellArrayMap cam = new CellArrayMap(getComparator(),cells,0,i,false); return new CellSet(cam); } - private CellSet createChunkBasedCellSet( + /*------------------------------------------------------------------------*/ + // Create CellSet based on CellArrayMap from current ConcurrentSkipListMap based CellSet + private CellSet recreateCellArrayMapSet(int numOfCells) { + + Cell[] cells = new Cell[numOfCells]; // build the Cell Array + Cell curCell; + int idx = 0; + // create this segment scanner with maximal possible read point, to go over all Cells + SegmentScanner segmentScanner = this.getSegmentScanner(Long.MAX_VALUE); + + try { + while ((curCell = segmentScanner.next()) != null) { + cells[idx++] = curCell; + } + } catch (IOException ie) { + throw new IllegalStateException(ie); + } + // build the immutable CellSet + CellArrayMap cam = new CellArrayMap(getComparator(),cells,0,idx,false); + return new CellSet(cam); + } + + /*------------------------------------------------------------------------*/ + // Create CellSet based on CellChunkMap from compacting iterator + // we do not consider cells bigger than chunks + private CellSet createCellChunkMapSet( int numOfCells, MemStoreCompactorIterator iterator, final Configuration conf) { + // calculate how many chunks we will need for metadata int chunkSize = conf.getInt(HeapMemStoreLAB.CHUNK_SIZE_KEY, HeapMemStoreLAB.CHUNK_SIZE_DEFAULT); - int numOfCellsInsideChunk = chunkSize / CellChunkMap.BYTES_IN_CELL; - int numberOfChunks = chunkSize/numOfCellsInsideChunk; - HeapMemStoreLAB ms = (HeapMemStoreLAB)getMemStoreLAB(); + int numOfCellsInChunk = chunkSize / CellChunkMap.BYTES_IN_CELL; + int numberOfChunks = numOfCells/numOfCellsInChunk; - // all Chunks must be allocated from current MSLAB - HeapMemStoreLAB.Chunk[] chunks = new HeapMemStoreLAB.Chunk[numberOfChunks]; + // all Chunks (for metadata and for data) are allocated from the current segment's MSLAB + // TODO: when Chunk is going to be out of HeapMemStoreLAB we can use MemStoreLAB here + // and not HeapMemStoreLAB + HeapMemStoreLAB ms = (HeapMemStoreLAB)getMemStoreLAB(); + HeapMemStoreLAB.Chunk[] chunks = new HeapMemStoreLAB.Chunk[numberOfChunks]; // metadata chunks int currentChunkIdx = 0; chunks[currentChunkIdx] = ms.allocateChunk(); int offsetInCurentChunk = 0; @@ -130,8 +211,7 @@ private CellSet createChunkBasedCellSet( Cell c = iterator.next(); if (offsetInCurentChunk + CellChunkMap.BYTES_IN_CELL > chunkSize) { - // we do not consider cells bigger than chunks - // continue to the next chunk + // continue to the next metadata chunk currentChunkIdx++; chunks[currentChunkIdx] = ms.allocateChunk(); offsetInCurentChunk = 0; @@ -142,6 +222,8 @@ private CellSet createChunkBasedCellSet( KeyValue kv = KeyValueUtil.ensureKeyValue(c); offsetInCurentChunk = cloneAndReference(kv, chunks[currentChunkIdx].getData(), offsetInCurentChunk); + // last parameter false, because in compaction count both Heap (Data) and MetaData size + updateMetaInfo(c,true); } CellChunkMap ccm = new CellChunkMap(getComparator(),(HeapMemStoreLAB)getMemStoreLAB(), @@ -149,11 +231,9 @@ private CellSet createChunkBasedCellSet( return new CellSet(ccm); } - /** - * If the segment has a memory allocator the cell is being cloned to this space, and returned; - * otherwise the given cell is returned - * @return either the given cell or its clone - */ + /*------------------------------------------------------------------------*/ + // for a given cell, allocate space and write the cell on the data chunk, + // then write the cell-reference on the metadata chunk private int cloneAndReference(Cell cell, byte[] referencesByteArray, int offsetForReference) { HeapMemStoreLAB ms = (HeapMemStoreLAB)getMemStoreLAB(); int len = KeyValueUtil.length(cell); @@ -161,8 +241,9 @@ private int cloneAndReference(Cell cell, byte[] referencesByteArray, int offsetF // we assume Cell length is not bigger than Chunk // allocate - ByteRange alloc = ms.allocateBytes(len); - int chunkId = ms.getCurrentChunkId(); + Pair tuple = ms.allocateBytesWithID(len); + ByteRange alloc = tuple.getFirst(); + int chunkId = tuple.getSecond(); KeyValueUtil.appendToByteArray(cell, alloc.getBytes(), alloc.getOffset()); // write the reference diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java index 32f3f034cf48..1faa43afbfb8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java @@ -83,7 +83,8 @@ public class MemStoreChunkPool { private AtomicLong reusedChunkCount = new AtomicLong(); private AtomicInteger chunkIDs = new AtomicInteger(1); // 14921 - // 14921: IDs Mapping of all chunks (key 0 is forbidden) + // The mapping between each chunk allocated by MemStoreChunkPool and + // an integer representing the chunk's ID (ID 0 is forbidden) private final ConcurrentMap chunksMap = new ConcurrentHashMap(); MemStoreChunkPool(Configuration conf, int chunkSize, int maxCount, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorIterator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorIterator.java index 506671d61213..56c261934ddb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorIterator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorIterator.java @@ -39,6 +39,7 @@ public class MemStoreCompactorIterator implements Iterator { private List kvs = new ArrayList(); // scanner for full or partial pipeline (heap of segment scanners) + // we need to keep those scanners in order to close them at the end private KeyValueScanner scanner; // scanner on top of pipeline scanner that uses ScanQueryMatcher @@ -50,7 +51,7 @@ public class MemStoreCompactorIterator implements Iterator { private Iterator kvsIterator; // C-tor - public MemStoreCompactorIterator(int initiator, LinkedList segments, + public MemStoreCompactorIterator(LinkedList segments, CellComparator comparator, int compactionKVMax, HStore store) throws IOException { this.scannerContext = ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build(); @@ -67,7 +68,7 @@ public MemStoreCompactorIterator(int initiator, LinkedList seg scanner = new MemStoreScanner(comparator, scanners, MemStoreScanner.Type.COMPACT_FORWARD); // reinitialize the compacting scanner for each instance of iterator - compactingScanner = createScanner(store, scanner, initiator); + compactingScanner = createScanner(store, scanner); hasMore = compactingScanner.next(kvs, scannerContext); @@ -80,6 +81,7 @@ public MemStoreCompactorIterator(int initiator, LinkedList seg @Override public boolean hasNext() { if (!kvsIterator.hasNext()) { + // refillKVS() method should be invoked only if !kvsIterator.hasNext() if (!refillKVS()) { return false; } @@ -87,34 +89,33 @@ public boolean hasNext() { return hasMore; } - - @Override public Cell next() { if (!kvsIterator.hasNext()) { + // refillKVS() method should be invoked only if !kvsIterator.hasNext() if (!refillKVS()) return null; } return (!hasMore) ? null : kvsIterator.next(); } - - - @Override - public void remove() { + public void close() { compactingScanner.close(); compactingScanner = null; scanner.close(); scanner = null; } - + @Override + public void remove() { + throw new UnsupportedOperationException(); + } /** * Creates the scanner for compacting the pipeline. * * @return the scanner */ - private StoreScanner createScanner(Store store, KeyValueScanner scanner, int init) + private StoreScanner createScanner(Store store, KeyValueScanner scanner) throws IOException { Scan scan = new Scan(); @@ -144,9 +145,8 @@ private boolean refillKVS() { if (!kvs.isEmpty() ) { // is the new KVS empty ? kvsIterator = kvs.iterator(); return true; - } else { - return false; } + return false; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java index 63db3b5e0a98..05928f1d9f82 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java @@ -21,6 +21,7 @@ import java.util.Iterator; import java.util.SortedSet; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.logging.Log; import org.apache.hadoop.hbase.*; @@ -39,19 +40,19 @@ @InterfaceAudience.Private public abstract class Segment { - private volatile CellSet cellSet; + private AtomicReference cellSet= new AtomicReference(); private final CellComparator comparator; private long minSequenceId; private volatile MemStoreLAB memStoreLAB; private final AtomicLong size; private final TimeRangeTracker timeRangeTracker; - private long constantCellMetaDataSize; + protected long constantCellMetaDataSize; protected volatile boolean tagsPresent; protected Segment( CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, long size, long constantCellSize) { - this.cellSet = cellSet; + this.cellSet.set(cellSet); this.comparator = comparator; this.minSequenceId = Long.MAX_VALUE; this.memStoreLAB = memStoreLAB; @@ -62,7 +63,7 @@ protected Segment( } protected Segment(Segment segment) { - this.cellSet = segment.getCellSet(); + this.cellSet.set(segment.getCellSet()); this.comparator = segment.getComparator(); this.minSequenceId = segment.getMinSequenceId(); this.memStoreLAB = segment.getMemStoreLAB(); @@ -186,8 +187,8 @@ public Segment setSize(long size) { * @return this object */ - protected Segment setCellSet(CellSet cellSet) { - this.cellSet = cellSet; + protected Segment setCellSet(CellSet cellSetOld, CellSet cellSetNew) { + this.cellSet.compareAndSet(cellSetOld,cellSetNew); return this; } @@ -240,7 +241,7 @@ public int compareRows(Cell left, Cell right) { * @return a set of all cells in the segment */ protected CellSet getCellSet() { - return cellSet; + return cellSet.get(); } /** @@ -253,12 +254,12 @@ protected CellComparator getComparator() { protected long internalAdd(Cell cell) { boolean succ = getCellSet().add(cell); - long s = updateMetaInfo(cell, succ, false); + long s = updateMetaInfo(cell, succ); return s; } - protected long updateMetaInfo(Cell cellToAdd, boolean succ, boolean flattenning) { - long s = flattenning ? constantCellMetaDataSize : heapSizeChange(cellToAdd, succ); + protected long updateMetaInfo(Cell cellToAdd, boolean succ) { + long s = heapSizeChange(cellToAdd, succ); getTimeRangeTracker().includeTimestamp( cellToAdd); size.addAndGet(s); minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java index 7b11c08b89e1..038bf307cf43 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java @@ -19,14 +19,12 @@ package org.apache.hadoop.hbase.regionserver; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.ReflectionUtils; import java.io.IOException; -import java.util.List; /** * A singleton store segment factory. @@ -45,15 +43,15 @@ public static SegmentFactory instance() { return instance; } - // create skip-list-based immutable segment + // create skip-list-based (non-flat) immutable segment from compacting old immutable segments public ImmutableSegment createImmutableSegment(final Configuration conf, - final CellComparator comparator, long size) { + final CellComparator comparator, MemStoreCompactorIterator iterator) { MemStoreLAB memStoreLAB = getMemStoreLAB(conf); - MutableSegment segment = generateMutableSegment(conf, comparator, memStoreLAB, size); - return createImmutableSegment(segment); + return + new ImmutableSegment(comparator, iterator, memStoreLAB); } - // usually used to create empty immutable segment + // create empty immutable segment public ImmutableSegment createImmutableSegment(CellComparator comparator, long size) { MutableSegment segment = generateMutableSegment(null, comparator, null, size); return createImmutableSegment(segment); @@ -71,27 +69,14 @@ public MutableSegment createMutableSegment(final Configuration conf, return generateMutableSegment(conf, comparator, memStoreLAB, size); } - // create flat immutable segment from skip-list-based old immutable segment - public ImmutableSegment createImmutableSegment(int numOfCells, ImmutableSegment oldSegment, - MemStoreCompactorIterator iterator) - throws IOException { - - // new Segment is using the MSLAB of the previous old Segment, because the MSLAB Chunks - // remain the same in the flattening process - return - new ImmutableSegment( - oldSegment, iterator, oldSegment.getMemStoreLAB(), numOfCells, ClassSize.CELL_ARRAY_ENTRY); - } - // create new flat immutable segment from compacting old immutable segment - public ImmutableSegment createImmutableSegment( - final Configuration conf, final CellComparator comparator, - int numOfCells, MemStoreCompactorIterator iterator, boolean array) + public ImmutableSegment createImmutableSegment(final Configuration conf, final CellComparator comparator, + MemStoreCompactorIterator iterator, int numOfCells, boolean array) throws IOException { MemStoreLAB memStoreLAB = getMemStoreLAB(conf); return new ImmutableSegment( - conf, comparator, iterator, memStoreLAB, numOfCells, ClassSize.CELL_ARRAY_ENTRY, array); + conf, comparator, iterator, memStoreLAB, numOfCells, array); } //****** private methods to instantiate concrete store segments **********// diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java index c622a98609b0..505ccf4ce2c5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java @@ -38,8 +38,7 @@ public class VersionedSegmentsList { private final LinkedList storeSegments; private final long version; - public VersionedSegmentsList( - LinkedList storeSegments, long version) { + public VersionedSegmentsList(LinkedList storeSegments, long version) { this.storeSegments = storeSegments; this.version = version; } @@ -60,10 +59,4 @@ public int getNumOfCells() { return totalCells; } - public ImmutableSegment getSkipListSegment() { - for (ImmutableSegment s : storeSegments) { - if (!s.isFlat()) return s; - } - return null; - } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java index 5c79d7257f05..7d4b52ef11ff 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java @@ -240,7 +240,7 @@ public void testNativeSizes() throws IOException { // CellSet cl = CellSet.class; expected = ClassSize.estimateBase(cl, false); - actual = ClassSize.CELL_SKIPLIST_SET; + actual = ClassSize.CELL_SET; if (expected != actual) { ClassSize.estimateBase(cl, true); assertEquals(expected, actual); From b9d84eb161907fe1a9ee87be5257f18b37198ee3 Mon Sep 17 00:00:00 2001 From: anastas Date: Thu, 14 Apr 2016 15:32:40 +0300 Subject: [PATCH 7/9] Before submitting the patch --- .../hbase/regionserver/CellArrayMap.java | 2 +- .../hbase/regionserver/CellChunkMap.java | 2 +- .../hbase/regionserver/CellFlatMap.java | 10 +- .../regionserver/CompactingMemStore.java | 20 +- .../regionserver/CompactionPipeline.java | 4 +- .../hbase/regionserver/HeapMemStoreLAB.java | 14 +- .../hbase/regionserver/ImmutableSegment.java | 38 +- .../hbase/regionserver/MemStoreChunkPool.java | 4 +- .../hbase/regionserver/SegmentFactory.java | 7 +- .../regionserver/TestCompactingMemStore.java | 25 +- .../TestCompactingMemStoreAB.java | 723 ------------------ .../TestCompactingToCellArrayMapMemStore.java | 242 ++++++ 12 files changed, 322 insertions(+), 769 deletions(-) delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStoreAB.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java index 8cc8b206057a..5111ff9e45ce 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java @@ -40,7 +40,7 @@ public CellArrayMap(Comparator comparator, Cell[] b, int min, int /* To be used by base class only to create a sub-CellFlatMap */ @Override - protected CellFlatMap createCellFlatMap(Comparator comparator, int min, int max, + protected CellFlatMap createSubCellFlatMap(Comparator comparator, int min, int max, boolean d) { return new CellArrayMap(comparator,this.block,min,max,d); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java index ec53e75aa6b8..4fa678ff9169 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java @@ -71,7 +71,7 @@ public CellChunkMap(Comparator comparator, HeapMemStoreLAB memStor /* To be used by base class only to create a sub-CellFlatMap */ @Override - protected CellFlatMap createCellFlatMap(Comparator comparator, int min, int max, + protected CellFlatMap createSubCellFlatMap(Comparator comparator, int min, int max, boolean d) { return new CellChunkMap(comparator, this.memStoreLAB, this.chunks, min, max, this.numOfCellsInsideChunk* BYTES_IN_CELL, d); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java index a209abaeedf5..f50fbb995cfc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java @@ -56,7 +56,7 @@ public CellFlatMap(Comparator comparator, int min, int max, boolea } /* Used for abstract CellFlatMap creation, implemented by derived class */ - protected abstract CellFlatMap createCellFlatMap(Comparator comparator, int min, + protected abstract CellFlatMap createSubCellFlatMap(Comparator comparator, int min, int max, boolean descending); /* Returns the i-th cell in the cell block */ @@ -132,24 +132,24 @@ public ConcurrentNavigableMap subMap( Cell fromKey, if (fromIndex > toIndex) { throw new IllegalArgumentException("inconsistent range"); } - return createCellFlatMap(comparator, fromIndex, toIndex, descending); + return createSubCellFlatMap(comparator, fromIndex, toIndex, descending); } @Override public ConcurrentNavigableMap headMap(Cell toKey, boolean inclusive) { int index = getValidIndex(toKey, inclusive); - return createCellFlatMap(comparator, minCellIdx, index, descending); + return createSubCellFlatMap(comparator, minCellIdx, index, descending); } @Override public ConcurrentNavigableMap tailMap(Cell fromKey, boolean inclusive) { int index = (getValidIndex(fromKey, !inclusive)); - return createCellFlatMap(comparator, index, maxCellIdx, descending); + return createSubCellFlatMap(comparator, index, maxCellIdx, descending); } @Override public ConcurrentNavigableMap descendingMap() { - return createCellFlatMap(comparator, minCellIdx, maxCellIdx, true); + return createSubCellFlatMap(comparator, minCellIdx, maxCellIdx, true); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index 799e31fc0834..657fbe2e6391 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -332,7 +332,7 @@ private ExecutorService getPool() { } private boolean shouldFlushInMemory() { - if(getActive().getSize() > IN_MEMORY_FLUSH_THRESHOLD_FACTOR *flushSizeLowerBound) { + if(getActive().getSize() > IN_MEMORY_FLUSH_THRESHOLD_FACTOR*flushSizeLowerBound) { // size above flush threshold return (allowCompaction.get() && !inMemoryFlushInProgress.get()); } @@ -397,7 +397,9 @@ public void process() throws IOException { /** ---------------------------------------------------------------------- * The ongoing MemStore Compaction manager, dispatches a solo running compaction and interrupts - * the compaction if requested. Prior to compaction the MemStoreCompactor evaluates + * the compaction if requested. The compaction is interrupted and stopped by CompactingMemStore, + * for example when another compaction needs to be started. + * Prior to compaction the MemStoreCompactor evaluates * the compacting ratio and aborts the compaction if it is not worthy. * The MemStoreScanner is used to traverse the compaction pipeline. The MemStoreScanner * is included in internal store scanner, where all compaction logic is implemented. @@ -457,17 +459,17 @@ private void releaseResources() { private void doCompact() { int cellsAfterComp = versionedList.getNumOfCells(); try { - // Phase I (optional): estimate the compaction expedience - CHECK COMPACTION + // Phase I (optional): estimate the compaction expedience - EVALUATE COMPACTION if (COMPACTION_PRE_CHECK) { cellsAfterComp = countCellsForCompaction(); if (!isInterrupted.get() && (cellsAfterComp > COMPACTION_TRIGGER_REMAIN_FACTOR * versionedList.getNumOfCells())) { // too much cells "survive" the possible compaction we do not want to compact! - LOG.debug("Stopping the unworthy MemStore in-memory compaction for store " - + getFamilyName()); + LOG.debug("In-Memory compaction does not pay off - storing the flattened segment" + + " for store " + getFamilyName()); // Looking for Segment in the pipeline with SkipList index, to make it flat - pipeline.flattenOneSegment(versionedList.getVersion()); + pipeline.flattenYoungestSegment(versionedList.getVersion()); return; } } @@ -511,12 +513,14 @@ private ImmutableSegment compact(int numOfCells) case COMPACT_TO_ARRAY_MAP: result = SegmentFactory.instance() .createImmutableSegment( - getConfiguration(), getComparator(), iterator, numOfCells, true); + getConfiguration(), getComparator(), iterator, numOfCells, + ImmutableSegment.Type.ARRAY_MAP_BASED); break; case COMPACT_TO_CHUNK_MAP: result = SegmentFactory.instance() .createImmutableSegment( - getConfiguration(), getComparator(), iterator, numOfCells, false); + getConfiguration(), getComparator(), iterator, numOfCells, + ImmutableSegment.Type.CHUNK_MAP_BASED); break; default: throw new RuntimeException("Unknown type " + type); // sanity check } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java index fa51f79ad7f6..ede973ecfdd2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java @@ -120,14 +120,16 @@ public boolean swap(VersionedSegmentsList versionedList, ImmutableSegment segmen * * @return true iff a segment was successfully flattened */ - public boolean flattenOneSegment(long requesterVersion) { + public boolean flattenYoungestSegment(long requesterVersion) { if(requesterVersion != version) { + LOG.info("Segment flattening failed, because versions do not match"); return false; } synchronized (pipeline){ if(requesterVersion != version) { + LOG.info("Segment flattening failed, because versions do not match"); return false; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java index 63617251461d..6ca0f372c0e9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java @@ -112,7 +112,7 @@ public Pair allocateBytesWithID(int size) { // Callers should satisfy large allocations directly from JVM since they // don't cause fragmentation as badly. if (size > maxAlloc) { - return null; + return new Pair<>(null,0); } while (true) { @@ -220,7 +220,7 @@ private Chunk getOrMakeChunk() { } } - /** HBASE-14921 + /** * Given a chunk ID return reference to the relevant chunk * @return a chunk */ @@ -228,7 +228,7 @@ public Chunk translateIdToChunk(int id) { return chunkPool.translateIdToChunk(id); } - /** HBASE-14921 + /** * Give the ID of the Chunk from where last allocation took the bytes * @return a chunk */ @@ -236,7 +236,7 @@ public int getCurrentChunkId() { return curChunk.get().getId(); } - /** HBASE-14921 + /** * Use instead of allocateBytes() when new full chunk is needed * @return a chunk */ @@ -267,7 +267,7 @@ static class Chunk { /** Size of chunk in bytes */ private final int size; - /* 14921: A unique identifier of a chunk inside MemStoreChunkPool */ + /* A unique identifier of a chunk inside MemStoreChunkPool */ private final int id; /* Chunk's index serves as replacement for pointer */ @@ -360,10 +360,10 @@ public String toString() { public int getId() { return id; - } // 14921 + } public byte[] getData() { return data; - } // 14921 + } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java index bfed35791e5c..3239bfd9c0b5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java @@ -43,17 +43,31 @@ @InterfaceAudience.Private public class ImmutableSegment extends Segment { - private boolean isFlat; // whether it is based on CellFlatMap or ConcurrentSkipListMap + /** + * Types of ImmutableSegment + */ + public enum Type { + SKIPLIST_MAP_BASED, + ARRAY_MAP_BASED, + CHUNK_MAP_BASED + } + + private Type type = Type.SKIPLIST_MAP_BASED; + + // whether it is based on CellFlatMap or ConcurrentSkipListMap + private boolean isFlat(){ + return (type == Type.ARRAY_MAP_BASED) || (type == Type.CHUNK_MAP_BASED); + } ///////////////////// CONSTRUCTORS ///////////////////// /**------------------------------------------------------------------------ - * C-tor to be used when new ImmutableSegment is being built from a Mutable one. + * Copy C-tor to be used when new ImmutableSegment is being built from a Mutable one. * This C-tor should be used when active MutableSegment is pushed into the compaction * pipeline and becomes an ImmutableSegment. */ protected ImmutableSegment(Segment segment) { super(segment); - isFlat = false; + type = Type.SKIPLIST_MAP_BASED; } /**------------------------------------------------------------------------ @@ -65,19 +79,21 @@ protected ImmutableSegment(Segment segment) { */ protected ImmutableSegment( final Configuration conf, CellComparator comparator, MemStoreCompactorIterator iterator, - MemStoreLAB memStoreLAB, int numOfCells, boolean array) { + MemStoreLAB memStoreLAB, int numOfCells, Type type) { super(null, comparator, memStoreLAB, CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_FLAT_ARRAY_ITEM, - array ? ClassSize.CELL_ARRAY_MAP_ENTRY : ClassSize.CELL_CHUNK_MAP_ENTRY); + (type == Type.ARRAY_MAP_BASED) ? + ClassSize.CELL_ARRAY_MAP_ENTRY : ClassSize.CELL_CHUNK_MAP_ENTRY); + CellSet cs = null; // build the CellSet Cell array or Byte array based - if (array) { + if (type == Type.ARRAY_MAP_BASED) { cs = createCellArrayMapSet(numOfCells, iterator); } else { cs = createCellChunkMapSet(numOfCells, iterator, conf); } this.setCellSet(null, cs); // update the CellSet of the new Segment - isFlat = true; + this.type = type; } /**------------------------------------------------------------------------ @@ -88,8 +104,8 @@ protected ImmutableSegment( protected ImmutableSegment( CellComparator comparator, MemStoreCompactorIterator iterator, MemStoreLAB memStoreLAB) { - super(null, comparator, memStoreLAB, CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_ITEM, - ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); + super(new CellSet(comparator), comparator, memStoreLAB, + CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_ITEM, ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); while (iterator.hasNext()) { Cell c = iterator.next(); @@ -99,7 +115,7 @@ protected ImmutableSegment( Cell newKV = maybeCloneWithAllocator(kv); internalAdd(newKV); } - isFlat = false; + type = Type.SKIPLIST_MAP_BASED; } ///////////////////// PUBLIC METHODS ///////////////////// @@ -126,7 +142,7 @@ public KeyValueScanner getKeyValueScanner() { * before the flattening and then replaced using CAS instruction. */ public boolean flatten() { - if (isFlat) return false; + if (isFlat()) return false; CellSet oldCellSet = getCellSet(); int numOfCells = getCellsCount(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java index 1faa43afbfb8..2da8aa4d0ad3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java @@ -118,7 +118,7 @@ Chunk getChunk() { createdChunkCount.incrementAndGet(); } else { chunk.reset(); - chunk.init(); // 14921 + chunk.init(); reusedChunkCount.incrementAndGet(); } return chunk; @@ -177,7 +177,7 @@ ConcurrentMap getChunksMap() { * Allocate and register Chunk */ private Chunk allocateChunk() { - int newId = chunkIDs.getAndAdd(1); // the number of the new chunk + int newId = chunkIDs.getAndAdd(1); // the id of the new chunk Chunk chunk = new Chunk(chunkSize,newId); chunksMap.put(newId, chunk); chunk.init(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java index 038bf307cf43..6f923618b6b3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java @@ -18,6 +18,7 @@ */ package org.apache.hadoop.hbase.regionserver; +import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -71,12 +72,14 @@ public MutableSegment createMutableSegment(final Configuration conf, // create new flat immutable segment from compacting old immutable segment public ImmutableSegment createImmutableSegment(final Configuration conf, final CellComparator comparator, - MemStoreCompactorIterator iterator, int numOfCells, boolean array) + MemStoreCompactorIterator iterator, int numOfCells, ImmutableSegment.Type segmentType) throws IOException { + Preconditions.checkArgument( + segmentType != ImmutableSegment.Type.SKIPLIST_MAP_BASED, "wrong immutable segment type"); MemStoreLAB memStoreLAB = getMemStoreLAB(conf); return new ImmutableSegment( - conf, comparator, iterator, memStoreLAB, numOfCells, array); + conf, comparator, iterator, memStoreLAB, numOfCells, segmentType); } //****** private methods to instantiate concrete store segments **********// diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java index f84de134521e..0dacb7ef5d55 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java @@ -61,15 +61,15 @@ public class TestCompactingMemStore extends TestDefaultMemStore { private static final Log LOG = LogFactory.getLog(TestCompactingMemStore.class); - private static MemStoreChunkPool chunkPool; - private HRegion region; - private RegionServicesForStores regionServicesForStores; - private HStore store; + protected static MemStoreChunkPool chunkPool; + protected HRegion region; + protected RegionServicesForStores regionServicesForStores; + protected HStore store; ////////////////////////////////////////////////////////////////////////////// // Helpers ////////////////////////////////////////////////////////////////////////////// - private static byte[] makeQualifier(final int i1, final int i2) { + protected static byte[] makeQualifier(final int i1, final int i2) { return Bytes.toBytes(Integer.toString(i1) + ";" + Integer.toString(i2)); } @@ -81,6 +81,12 @@ public void tearDown() throws Exception { @Override public void setUp() throws Exception { + compactingSetUp(); + this.memstore = new CompactingMemStore(HBaseConfiguration.create(), CellComparator.COMPARATOR, + store, regionServicesForStores); + } + + protected void compactingSetUp() throws Exception { super.internalSetUp(); Configuration conf = new Configuration(); conf.setBoolean(SegmentFactory.USEMSLAB_KEY, true); @@ -91,13 +97,11 @@ public void setUp() throws Exception { this.region = hbaseUtility.createTestRegion("foobar", hcd); this.regionServicesForStores = region.getRegionServicesForStores(); this.store = new HStore(region, hcd, conf); - this.memstore = new CompactingMemStore(HBaseConfiguration.create(), CellComparator.COMPARATOR, - store, regionServicesForStores); + chunkPool = MemStoreChunkPool.getPool(conf); assertTrue(chunkPool != null); } - /** * A simple test which verifies the 3 possible states when scanning across snapshot. * @@ -609,6 +613,11 @@ public void testCompaction2Buckets() throws IOException { while (((CompactingMemStore)memstore).isMemStoreFlushingInMemory()) { Threads.sleep(1000); } + int counter = 0; + for ( Segment s : memstore.getListOfSegments()) { + counter += s.getCellsCount(); + } + assertEquals(3, counter); assertEquals(0, memstore.getSnapshot().getCellsCount()); assertEquals(528, regionServicesForStores.getGlobalMemstoreTotalSize()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStoreAB.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStoreAB.java deleted file mode 100644 index c91ea4085d01..000000000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStoreAB.java +++ /dev/null @@ -1,723 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.testclassification.RegionServerTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdge; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.Threads; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import java.io.IOException; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.util.ArrayList; -import java.util.List; - -/** - * compacted memstore test case - */ -@Category({RegionServerTests.class, MediumTests.class}) -public class TestCompactingMemStoreAB extends TestCompactingMemStore { - - private static final Log LOG = LogFactory.getLog(TestCompactingMemStoreAB.class); - private static MemStoreChunkPool chunkPool; - private HRegion region; - private RegionServicesForStores regionServicesForStores; - private HStore store; - - ////////////////////////////////////////////////////////////////////////////// - // Helpers - ////////////////////////////////////////////////////////////////////////////// - private static byte[] makeQualifier(final int i1, final int i2) { - return Bytes.toBytes(Integer.toString(i1) + ";" + - Integer.toString(i2)); - } - - @Override public void tearDown() throws Exception { - chunkPool.clearChunks(); - } - - @Override public void setUp() throws Exception { - super.internalSetUp(); - Configuration conf = new Configuration(); - conf.setBoolean(SegmentFactory.USEMSLAB_KEY, true); - conf.setFloat(MemStoreChunkPool.CHUNK_POOL_MAXSIZE_KEY, 0.2f); - conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000); - HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf); - HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); - this.region = hbaseUtility.createTestRegion("foobar", hcd); - this.regionServicesForStores = region.getRegionServicesForStores(); - this.store = new HStore(region, hcd, conf); - this.memstore = - new CompactingMemStore(HBaseConfiguration.create(), CellComparator.COMPARATOR, store, - regionServicesForStores, CompactingMemStore.Type.COMPACT_TO_ARRAY_MAP); - chunkPool = MemStoreChunkPool.getPool(conf); - assertTrue(chunkPool != null); - } - - /** - * A simple test which verifies the 3 possible states when scanning across snapshot. - * - * @throws IOException - * @throws CloneNotSupportedException - */ - public void testScanAcrossSnapshot2() throws IOException, CloneNotSupportedException { - // we are going to the scanning across snapshot with two kvs - // kv1 should always be returned before kv2 - final byte[] one = Bytes.toBytes(1); - final byte[] two = Bytes.toBytes(2); - final byte[] f = Bytes.toBytes("f"); - final byte[] q = Bytes.toBytes("q"); - final byte[] v = Bytes.toBytes(3); - - final KeyValue kv1 = new KeyValue(one, f, q, 10, v); - final KeyValue kv2 = new KeyValue(two, f, q, 10, v); - - // use case 1: both kvs in kvset - this.memstore.add(kv1.clone()); - this.memstore.add(kv2.clone()); - verifyScanAcrossSnapshot2(kv1, kv2); - - // use case 2: both kvs in snapshot - this.memstore.snapshot(0); - verifyScanAcrossSnapshot2(kv1, kv2); - - // use case 3: first in snapshot second in kvset - this.memstore = - new CompactingMemStore(HBaseConfiguration.create(), CellComparator.COMPARATOR, store, - regionServicesForStores); - this.memstore.add(kv1.clone()); - // As compaction is starting in the background the repetition - // of the k1 might be removed BUT the scanners created earlier - // should look on the OLD MutableCellSetSegment, so this should be OK... - this.memstore.snapshot(0); - this.memstore.add(kv2.clone()); - verifyScanAcrossSnapshot2(kv1, kv2); - } - - /** - * Test memstore snapshots - * - * @throws IOException - */ - public void testSnapshotting() throws IOException { - final int snapshotCount = 5; - // Add some rows, run a snapshot. Do it a few times. - for (int i = 0; i < snapshotCount; i++) { - addRows(this.memstore); - runSnapshot(this.memstore, true); - assertEquals("History not being cleared", 0, this.memstore.getSnapshot().getCellsCount()); - } - } - - ////////////////////////////////////////////////////////////////////////////// - // Get tests - ////////////////////////////////////////////////////////////////////////////// - - /** - * Test getNextRow from memstore - * - * @throws InterruptedException - */ - public void testGetNextRow() throws Exception { - addRows(this.memstore); - // Add more versions to make it a little more interesting. - Thread.sleep(1); - addRows(this.memstore); - Cell closestToEmpty = ((CompactingMemStore) this.memstore).getNextRow(KeyValue.LOWESTKEY); - assertTrue(KeyValue.COMPARATOR - .compareRows(closestToEmpty, new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0); - for (int i = 0; i < ROW_COUNT; i++) { - Cell nr = ((CompactingMemStore) this.memstore) - .getNextRow(new KeyValue(Bytes.toBytes(i), System.currentTimeMillis())); - if (i + 1 == ROW_COUNT) { - assertEquals(nr, null); - } else { - assertTrue(KeyValue.COMPARATOR - .compareRows(nr, new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0); - } - } - //starting from each row, validate results should contain the starting row - Configuration conf = HBaseConfiguration.create(); - for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) { - ScanInfo scanInfo = - new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE, 0, this.memstore.getComparator()); - ScanType scanType = ScanType.USER_SCAN; - InternalScanner scanner = - new StoreScanner(new Scan(Bytes.toBytes(startRowId)), scanInfo, scanType, null, memstore.getScanners(0)); - List results = new ArrayList(); - for (int i = 0; scanner.next(results); i++) { - int rowId = startRowId + i; - Cell left = results.get(0); - byte[] row1 = Bytes.toBytes(rowId); - assertTrue("Row name", - CellComparator.COMPARATOR.compareRows(left, row1, 0, row1.length) == 0); - assertEquals("Count of columns", QUALIFIER_COUNT, results.size()); - List row = new ArrayList(); - for (Cell kv : results) { - row.add(kv); - } - isExpectedRowWithoutTimestamps(rowId, row); - // Clear out set. Otherwise row results accumulate. - results.clear(); - } - } - } - - public void testGet_memstoreAndSnapShot() throws IOException { - byte[] row = Bytes.toBytes("testrow"); - byte[] fam = Bytes.toBytes("testfamily"); - byte[] qf1 = Bytes.toBytes("testqualifier1"); - byte[] qf2 = Bytes.toBytes("testqualifier2"); - byte[] qf3 = Bytes.toBytes("testqualifier3"); - byte[] qf4 = Bytes.toBytes("testqualifier4"); - byte[] qf5 = Bytes.toBytes("testqualifier5"); - byte[] val = Bytes.toBytes("testval"); - - //Setting up memstore - memstore.add(new KeyValue(row, fam, qf1, val)); - memstore.add(new KeyValue(row, fam, qf2, val)); - memstore.add(new KeyValue(row, fam, qf3, val)); - //Pushing to pipeline - ((CompactingMemStore) memstore).flushInMemory(); - assertEquals(0, memstore.getSnapshot().getCellsCount()); - //Creating a snapshot - memstore.snapshot(); - assertEquals(3, memstore.getSnapshot().getCellsCount()); - //Adding value to "new" memstore - assertEquals(0, memstore.getActive().getCellsCount()); - memstore.add(new KeyValue(row, fam, qf4, val)); - memstore.add(new KeyValue(row, fam, qf5, val)); - assertEquals(2, memstore.getActive().getCellsCount()); - } - - //////////////////////////////////// - //Test for upsert with MSLAB - //////////////////////////////////// - - /** - * Test a pathological pattern that shows why we can't currently - * use the MSLAB for upsert workloads. This test inserts data - * in the following pattern: - * - row0001 through row1000 (fills up one 2M Chunk) - * - row0002 through row1001 (fills up another 2M chunk, leaves one reference - * to the first chunk - * - row0003 through row1002 (another chunk, another dangling reference) - * This causes OOME pretty quickly if we use MSLAB for upsert - * since each 2M chunk is held onto by a single reference. - */ - public void testUpsertMSLAB() throws Exception { - - int ROW_SIZE = 2048; - byte[] qualifier = new byte[ROW_SIZE - 4]; - - MemoryMXBean bean = ManagementFactory.getMemoryMXBean(); - for (int i = 0; i < 3; i++) { - System.gc(); - } - long usageBefore = bean.getHeapMemoryUsage().getUsed(); - - long size = 0; - long ts = 0; - - for (int newValue = 0; newValue < 1000; newValue++) { - for (int row = newValue; row < newValue + 1000; row++) { - byte[] rowBytes = Bytes.toBytes(row); - size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts); - } - } - System.out.println("Wrote " + ts + " vals"); - for (int i = 0; i < 3; i++) { - System.gc(); - } - long usageAfter = bean.getHeapMemoryUsage().getUsed(); - System.out.println( - "Memory used: " + (usageAfter - usageBefore) + " (heapsize: " + memstore.heapSize() + - " size: " + size + ")"); - } - - //////////////////////////////////// - // Test for periodic memstore flushes - // based on time of oldest edit - //////////////////////////////////// - - /** - * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased - * as older keyvalues are deleted from the memstore. - * - * @throws Exception - */ - public void testUpsertMemstoreSize() throws Exception { - long oldSize = memstore.size(); - - List l = new ArrayList(); - KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v"); - KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v"); - KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v"); - - kv1.setSequenceId(1); - kv2.setSequenceId(1); - kv3.setSequenceId(1); - l.add(kv1); - l.add(kv2); - l.add(kv3); - - this.memstore.upsert(l, 2);// readpoint is 2 - long newSize = this.memstore.size(); - assert (newSize > oldSize); - //The kv1 should be removed. - assert (memstore.getActive().getCellsCount() == 2); - - KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v"); - kv4.setSequenceId(1); - l.clear(); - l.add(kv4); - this.memstore.upsert(l, 3); - assertEquals(newSize, this.memstore.size()); - //The kv2 should be removed. - assert (memstore.getActive().getCellsCount() == 2); - //this.memstore = null; - } - - /** - * Tests that the timeOfOldestEdit is updated correctly for the - * various edit operations in memstore. - * - * @throws Exception - */ - public void testUpdateToTimeOfOldestEdit() throws Exception { - try { - EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest(); - EnvironmentEdgeManager.injectEdge(edge); - long t = memstore.timeOfOldestEdit(); - assertEquals(t, Long.MAX_VALUE); - - // test the case that the timeOfOldestEdit is updated after a KV add - memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v")); - t = memstore.timeOfOldestEdit(); - assertTrue(t == 1234); - // The method will also assert - // the value is reset to Long.MAX_VALUE - t = runSnapshot(memstore, true); - - // test the case that the timeOfOldestEdit is updated after a KV delete - memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v")); - t = memstore.timeOfOldestEdit(); - assertTrue(t == 1234); - t = runSnapshot(memstore, true); - - // test the case that the timeOfOldestEdit is updated after a KV upsert - List l = new ArrayList(); - KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v"); - kv1.setSequenceId(100); - l.add(kv1); - memstore.upsert(l, 1000); - t = memstore.timeOfOldestEdit(); - assertTrue(t == 1234); - } finally { - EnvironmentEdgeManager.reset(); - } - } - - private long runSnapshot(final AbstractMemStore hmc, boolean useForce) throws IOException { - // Save off old state. - long oldHistorySize = hmc.getSnapshot().getSize(); - long prevTimeStamp = hmc.timeOfOldestEdit(); - - hmc.snapshot(0); - MemStoreSnapshot snapshot = hmc.snapshot(0); - if (useForce) { - // Make some assertions about what just happened. - assertTrue("History size has not increased", oldHistorySize < snapshot.getSize()); - long t = hmc.timeOfOldestEdit(); - assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE); - hmc.clearSnapshot(snapshot.getId()); - } else { - long t = hmc.timeOfOldestEdit(); - assertTrue("Time of oldest edit didn't remain the same", t == prevTimeStamp); - } - return prevTimeStamp; - } - - private void isExpectedRowWithoutTimestamps(final int rowIndex, List kvs) { - int i = 0; - for (Cell kv : kvs) { - byte[] expectedColname = makeQualifier(rowIndex, i++); - assertTrue("Column name", CellUtil.matchingQualifier(kv, expectedColname)); - // Value is column name as bytes. Usually result is - // 100 bytes in size at least. This is the default size - // for BytesWriteable. For comparison, convert bytes to - // String and trim to remove trailing null bytes. - assertTrue("Content", CellUtil.matchingValue(kv, expectedColname)); - } - } - - @Test public void testPuttingBackChunksAfterFlushing() throws IOException { - byte[] row = Bytes.toBytes("testrow"); - byte[] fam = Bytes.toBytes("testfamily"); - byte[] qf1 = Bytes.toBytes("testqualifier1"); - byte[] qf2 = Bytes.toBytes("testqualifier2"); - byte[] qf3 = Bytes.toBytes("testqualifier3"); - byte[] qf4 = Bytes.toBytes("testqualifier4"); - byte[] qf5 = Bytes.toBytes("testqualifier5"); - byte[] val = Bytes.toBytes("testval"); - - // Setting up memstore - memstore.add(new KeyValue(row, fam, qf1, val)); - memstore.add(new KeyValue(row, fam, qf2, val)); - memstore.add(new KeyValue(row, fam, qf3, val)); - - // Creating a snapshot - MemStoreSnapshot snapshot = memstore.snapshot(0); - assertEquals(3, memstore.getSnapshot().getCellsCount()); - - // Adding value to "new" memstore - assertEquals(0, memstore.getActive().getCellsCount()); - memstore.add(new KeyValue(row, fam, qf4, val)); - memstore.add(new KeyValue(row, fam, qf5, val)); - assertEquals(2, memstore.getActive().getCellsCount()); - memstore.clearSnapshot(snapshot.getId()); - - int chunkCount = chunkPool.getPoolSize(); - assertTrue(chunkCount > 0); - - } - - @Test public void testPuttingBackChunksWithOpeningScanner() throws IOException { - byte[] row = Bytes.toBytes("testrow"); - byte[] fam = Bytes.toBytes("testfamily"); - byte[] qf1 = Bytes.toBytes("testqualifier1"); - byte[] qf2 = Bytes.toBytes("testqualifier2"); - byte[] qf3 = Bytes.toBytes("testqualifier3"); - byte[] qf4 = Bytes.toBytes("testqualifier4"); - byte[] qf5 = Bytes.toBytes("testqualifier5"); - byte[] qf6 = Bytes.toBytes("testqualifier6"); - byte[] qf7 = Bytes.toBytes("testqualifier7"); - byte[] val = Bytes.toBytes("testval"); - - // Setting up memstore - memstore.add(new KeyValue(row, fam, qf1, val)); - memstore.add(new KeyValue(row, fam, qf2, val)); - memstore.add(new KeyValue(row, fam, qf3, val)); - - // Creating a snapshot - MemStoreSnapshot snapshot = memstore.snapshot(0); - assertEquals(3, memstore.getSnapshot().getCellsCount()); - - // Adding value to "new" memstore - assertEquals(0, memstore.getActive().getCellsCount()); - memstore.add(new KeyValue(row, fam, qf4, val)); - memstore.add(new KeyValue(row, fam, qf5, val)); - assertEquals(2, memstore.getActive().getCellsCount()); - - // opening scanner before clear the snapshot - List scanners = memstore.getScanners(0); - // Shouldn't putting back the chunks to pool,since some scanners are opening - // based on their data - memstore.clearSnapshot(snapshot.getId()); - - assertTrue(chunkPool.getPoolSize() == 0); - - // Chunks will be put back to pool after close scanners; - for (KeyValueScanner scanner : scanners) { - scanner.close(); - } - assertTrue(chunkPool.getPoolSize() > 0); - - // clear chunks - chunkPool.clearChunks(); - - // Creating another snapshot - - snapshot = memstore.snapshot(0); - // Adding more value - memstore.add(new KeyValue(row, fam, qf6, val)); - memstore.add(new KeyValue(row, fam, qf7, val)); - // opening scanners - scanners = memstore.getScanners(0); - // close scanners before clear the snapshot - for (KeyValueScanner scanner : scanners) { - scanner.close(); - } - // Since no opening scanner, the chunks of snapshot should be put back to - // pool - memstore.clearSnapshot(snapshot.getId()); - assertTrue(chunkPool.getPoolSize() > 0); - } - - @Test public void testPuttingBackChunksWithOpeningPipelineScanner() throws IOException { - byte[] row = Bytes.toBytes("testrow"); - byte[] fam = Bytes.toBytes("testfamily"); - byte[] qf1 = Bytes.toBytes("testqualifier1"); - byte[] qf2 = Bytes.toBytes("testqualifier2"); - byte[] qf3 = Bytes.toBytes("testqualifier3"); - byte[] val = Bytes.toBytes("testval"); - - // Setting up memstore - memstore.add(new KeyValue(row, fam, qf1, 1, val)); - memstore.add(new KeyValue(row, fam, qf2, 1, val)); - memstore.add(new KeyValue(row, fam, qf3, 1, val)); - - // Creating a pipeline - ((CompactingMemStore) memstore).disableCompaction(); - ((CompactingMemStore) memstore).flushInMemory(); - - // Adding value to "new" memstore - assertEquals(0, memstore.getActive().getCellsCount()); - memstore.add(new KeyValue(row, fam, qf1, 2, val)); - memstore.add(new KeyValue(row, fam, qf2, 2, val)); - assertEquals(2, memstore.getActive().getCellsCount()); - - // pipeline bucket 2 - ((CompactingMemStore) memstore).flushInMemory(); - // opening scanner before force flushing - List scanners = memstore.getScanners(0); - // Shouldn't putting back the chunks to pool,since some scanners are opening - // based on their data - ((CompactingMemStore) memstore).enableCompaction(); - // trigger compaction - ((CompactingMemStore) memstore).flushInMemory(); - - // Adding value to "new" memstore - assertEquals(0, memstore.getActive().getCellsCount()); - memstore.add(new KeyValue(row, fam, qf3, 3, val)); - memstore.add(new KeyValue(row, fam, qf2, 3, val)); - memstore.add(new KeyValue(row, fam, qf1, 3, val)); - assertEquals(3, memstore.getActive().getCellsCount()); - - while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { - Threads.sleep(10); - } - - assertTrue(chunkPool.getPoolSize() == 0); - - // Chunks will be put back to pool after close scanners; - for (KeyValueScanner scanner : scanners) { - scanner.close(); - } - assertTrue(chunkPool.getPoolSize() > 0); - - // clear chunks - chunkPool.clearChunks(); - - // Creating another snapshot - - MemStoreSnapshot snapshot = memstore.snapshot(0); - memstore.clearSnapshot(snapshot.getId()); - - snapshot = memstore.snapshot(0); - // Adding more value - memstore.add(new KeyValue(row, fam, qf2, 4, val)); - memstore.add(new KeyValue(row, fam, qf3, 4, val)); - // opening scanners - scanners = memstore.getScanners(0); - // close scanners before clear the snapshot - for (KeyValueScanner scanner : scanners) { - scanner.close(); - } - // Since no opening scanner, the chunks of snapshot should be put back to - // pool - memstore.clearSnapshot(snapshot.getId()); - assertTrue(chunkPool.getPoolSize() > 0); - } - - ////////////////////////////////////////////////////////////////////////////// - // Compaction tests - ////////////////////////////////////////////////////////////////////////////// - public void testCompaction1Bucket() throws IOException { - int counter = 0; - String[] keys1 = { "A", "A", "B", "C" }; //A1, A2, B3, C4 - - // test 1 bucket - addRowsByKeys(memstore, keys1); - assertEquals(704, regionServicesForStores.getGlobalMemstoreTotalSize()); - assertEquals(4, memstore.getActive().getCellsCount()); - long size = memstore.getFlushableSize(); - ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact - while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { - Threads.sleep(10); - } - assertEquals(0, memstore.getSnapshot().getCellsCount()); - assertEquals(344, regionServicesForStores.getGlobalMemstoreTotalSize()); - for ( Segment s : memstore.getListOfSegments()) { - counter += s.getCellsCount(); - } - assertEquals(3, counter); - size = memstore.getFlushableSize(); - MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot - region.addAndGetGlobalMemstoreSize(-size); // simulate flusher - ImmutableSegment s = memstore.getSnapshot(); - assertEquals(3, s.getCellsCount()); - assertEquals(0, regionServicesForStores.getGlobalMemstoreTotalSize()); - - memstore.clearSnapshot(snapshot.getId()); - } - - public void testCompaction2Buckets() throws IOException { - - String[] keys1 = { "A", "A", "B", "C" }; - String[] keys2 = { "A", "B", "D" }; - - addRowsByKeys(memstore, keys1); - assertEquals(704, regionServicesForStores.getGlobalMemstoreTotalSize()); - long size = memstore.getFlushableSize(); - -// assertTrue( -// "\n\n<<< This is the active size with 4 keys - " + memstore.getActive().getSize() -// + ". This is the memstore flushable size - " + size + "\n",false); - - ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact - while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { - Threads.sleep(1000); - } - int counter = 0; - for ( Segment s : memstore.getListOfSegments()) { - counter += s.getCellsCount(); - } - assertEquals(3,counter); - assertEquals(0, memstore.getSnapshot().getCellsCount()); - assertEquals(344, regionServicesForStores.getGlobalMemstoreTotalSize()); - - addRowsByKeys(memstore, keys2); - assertEquals(872, regionServicesForStores.getGlobalMemstoreTotalSize()); - - size = memstore.getFlushableSize(); - ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact - int i = 0; - while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { - Threads.sleep(10); - if (i > 10000000) { - ((CompactingMemStore) memstore).debug(); - assertTrue("\n\n<<< Infinite loop! :( \n", false); - } - } - assertEquals(0, memstore.getSnapshot().getCellsCount()); - counter = 0; - for ( Segment s : memstore.getListOfSegments()) { - counter += s.getCellsCount(); - } - assertEquals(4,counter); - assertEquals(480, regionServicesForStores.getGlobalMemstoreTotalSize()); - - size = memstore.getFlushableSize(); - MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot - region.addAndGetGlobalMemstoreSize(-size); // simulate flusher - ImmutableSegment s = memstore.getSnapshot(); - assertEquals(4, s.getCellsCount()); - assertEquals(0, regionServicesForStores.getGlobalMemstoreTotalSize()); - - memstore.clearSnapshot(snapshot.getId()); - } - - public void testCompaction3Buckets() throws IOException { - - String[] keys1 = { "A", "A", "B", "C" }; - String[] keys2 = { "A", "B", "D" }; - String[] keys3 = { "D", "B", "B" }; - - addRowsByKeys(memstore, keys1); - assertEquals(704, region.getMemstoreSize()); - - long size = memstore.getFlushableSize(); - ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact - - String tstStr = "\n\nFlushable size after first flush in memory:" + size + ". Is MemmStore in compaction?:" - + ((CompactingMemStore) memstore).isMemStoreFlushingInMemory(); - while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { - Threads.sleep(10); - } - assertEquals(0, memstore.getSnapshot().getCellsCount()); - assertEquals(344, regionServicesForStores.getGlobalMemstoreTotalSize()); - - addRowsByKeys(memstore, keys2); - - tstStr += " After adding second part of the keys. Memstore size: " + - region.getMemstoreSize() + ", Memstore Total Size: " + - regionServicesForStores.getGlobalMemstoreTotalSize() + "\n\n"; - - assertEquals(872, regionServicesForStores.getGlobalMemstoreTotalSize()); - - ((CompactingMemStore) memstore).disableCompaction(); - size = memstore.getFlushableSize(); - ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline without compaction - assertEquals(0, memstore.getSnapshot().getCellsCount()); - assertEquals(872, regionServicesForStores.getGlobalMemstoreTotalSize()); - - addRowsByKeys(memstore, keys3); - assertEquals(1400, regionServicesForStores.getGlobalMemstoreTotalSize()); - - ((CompactingMemStore) memstore).enableCompaction(); - size = memstore.getFlushableSize(); - ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact - while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { - Threads.sleep(10); - } - assertEquals(0, memstore.getSnapshot().getCellsCount()); - assertEquals(480, regionServicesForStores.getGlobalMemstoreTotalSize()); - - size = memstore.getFlushableSize(); - MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot - region.addAndGetGlobalMemstoreSize(-size); // simulate flusher - ImmutableSegment s = memstore.getSnapshot(); - assertEquals(4, s.getCellsCount()); - assertEquals(0, regionServicesForStores.getGlobalMemstoreTotalSize()); - - memstore.clearSnapshot(snapshot.getId()); - - //assertTrue(tstStr, false); - } - - private void addRowsByKeys(final AbstractMemStore hmc, String[] keys) { - byte[] fam = Bytes.toBytes("testfamily"); - byte[] qf = Bytes.toBytes("testqualifier"); - for (int i = 0; i < keys.length; i++) { - long timestamp = System.currentTimeMillis(); - Threads.sleep(1); // to make sure each kv gets a different ts - byte[] row = Bytes.toBytes(keys[i]); - byte[] val = Bytes.toBytes(keys[i] + i); - KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); - hmc.add(kv); - LOG.debug("added kv: " + kv.getKeyString() + ", timestamp" + kv.getTimestamp()); - long size = AbstractMemStore.heapSizeChange(kv, true); - regionServicesForStores.addAndGetGlobalMemstoreSize(size); - } - } - - private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge { - long t = 1234; - - @Override public long currentTime() { - return t; - } - - public void setCurrentTimeMillis(long t) { - this.t = t; - } - } - -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java new file mode 100644 index 000000000000..e2ae2534f7e4 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java @@ -0,0 +1,242 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.util.ArrayList; +import java.util.List; + +/** + * compacted memstore test case + */ +@Category({RegionServerTests.class, MediumTests.class}) +public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore { + + private static final Log LOG = LogFactory.getLog(TestCompactingToCellArrayMapMemStore.class); + //private static MemStoreChunkPool chunkPool; + //private HRegion region; + //private RegionServicesForStores regionServicesForStores; + //private HStore store; + + ////////////////////////////////////////////////////////////////////////////// + // Helpers + ////////////////////////////////////////////////////////////////////////////// + + @Override public void tearDown() throws Exception { + chunkPool.clearChunks(); + } + + @Override public void setUp() throws Exception { + compactingSetUp(); + this.memstore = + new CompactingMemStore(HBaseConfiguration.create(), CellComparator.COMPARATOR, store, + regionServicesForStores, CompactingMemStore.Type.COMPACT_TO_ARRAY_MAP); + } + + ////////////////////////////////////////////////////////////////////////////// + // Compaction tests + ////////////////////////////////////////////////////////////////////////////// + public void testCompaction1Bucket() throws IOException { + int counter = 0; + String[] keys1 = { "A", "A", "B", "C" }; //A1, A2, B3, C4 + + // test 1 bucket + addRowsByKeys(memstore, keys1); + assertEquals(704, regionServicesForStores.getGlobalMemstoreTotalSize()); + assertEquals(4, memstore.getActive().getCellsCount()); + long size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(440, regionServicesForStores.getGlobalMemstoreTotalSize()); + for ( Segment s : memstore.getListOfSegments()) { + counter += s.getCellsCount(); + } + assertEquals(3, counter); + size = memstore.getFlushableSize(); + MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot + region.addAndGetGlobalMemstoreSize(-size); // simulate flusher + ImmutableSegment s = memstore.getSnapshot(); + assertEquals(3, s.getCellsCount()); + assertEquals(0, regionServicesForStores.getGlobalMemstoreTotalSize()); + + memstore.clearSnapshot(snapshot.getId()); + } + + public void testCompaction2Buckets() throws IOException { + + String[] keys1 = { "A", "A", "B", "C" }; + String[] keys2 = { "A", "B", "D" }; + + addRowsByKeys(memstore, keys1); + assertEquals(704, regionServicesForStores.getGlobalMemstoreTotalSize()); + long size = memstore.getFlushableSize(); + +// assertTrue( +// "\n\n<<< This is the active size with 4 keys - " + memstore.getActive().getSize() +// + ". This is the memstore flushable size - " + size + "\n",false); + + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(1000); + } + int counter = 0; + for ( Segment s : memstore.getListOfSegments()) { + counter += s.getCellsCount(); + } + assertEquals(3,counter); + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(440, regionServicesForStores.getGlobalMemstoreTotalSize()); + + addRowsByKeys(memstore, keys2); + assertEquals(968, regionServicesForStores.getGlobalMemstoreTotalSize()); + + size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + int i = 0; + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + if (i > 10000000) { + ((CompactingMemStore) memstore).debug(); + assertTrue("\n\n<<< Infinite loop! :( \n", false); + } + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + counter = 0; + for ( Segment s : memstore.getListOfSegments()) { + counter += s.getCellsCount(); + } + assertEquals(4,counter); + assertEquals(592, regionServicesForStores.getGlobalMemstoreTotalSize()); + + size = memstore.getFlushableSize(); + MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot + region.addAndGetGlobalMemstoreSize(-size); // simulate flusher + ImmutableSegment s = memstore.getSnapshot(); + assertEquals(4, s.getCellsCount()); + assertEquals(0, regionServicesForStores.getGlobalMemstoreTotalSize()); + + memstore.clearSnapshot(snapshot.getId()); + } + + public void testCompaction3Buckets() throws IOException { + + String[] keys1 = { "A", "A", "B", "C" }; + String[] keys2 = { "A", "B", "D" }; + String[] keys3 = { "D", "B", "B" }; + + addRowsByKeys(memstore, keys1); + assertEquals(704, region.getMemstoreSize()); + + long size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + + String tstStr = "\n\nFlushable size after first flush in memory:" + size + ". Is MemmStore in compaction?:" + + ((CompactingMemStore) memstore).isMemStoreFlushingInMemory(); + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(440, regionServicesForStores.getGlobalMemstoreTotalSize()); + + addRowsByKeys(memstore, keys2); + + tstStr += " After adding second part of the keys. Memstore size: " + + region.getMemstoreSize() + ", Memstore Total Size: " + + regionServicesForStores.getGlobalMemstoreTotalSize() + "\n\n"; + + assertEquals(968, regionServicesForStores.getGlobalMemstoreTotalSize()); + + ((CompactingMemStore) memstore).disableCompaction(); + size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline without compaction + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(968, regionServicesForStores.getGlobalMemstoreTotalSize()); + + addRowsByKeys(memstore, keys3); + assertEquals(1496, regionServicesForStores.getGlobalMemstoreTotalSize()); + + ((CompactingMemStore) memstore).enableCompaction(); + size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(592, regionServicesForStores.getGlobalMemstoreTotalSize()); + + size = memstore.getFlushableSize(); + MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot + region.addAndGetGlobalMemstoreSize(-size); // simulate flusher + ImmutableSegment s = memstore.getSnapshot(); + assertEquals(4, s.getCellsCount()); + assertEquals(0, regionServicesForStores.getGlobalMemstoreTotalSize()); + + memstore.clearSnapshot(snapshot.getId()); + + //assertTrue(tstStr, false); + } + + private void addRowsByKeys(final AbstractMemStore hmc, String[] keys) { + byte[] fam = Bytes.toBytes("testfamily"); + byte[] qf = Bytes.toBytes("testqualifier"); + for (int i = 0; i < keys.length; i++) { + long timestamp = System.currentTimeMillis(); + Threads.sleep(1); // to make sure each kv gets a different ts + byte[] row = Bytes.toBytes(keys[i]); + byte[] val = Bytes.toBytes(keys[i] + i); + KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); + hmc.add(kv); + LOG.debug("added kv: " + kv.getKeyString() + ", timestamp" + kv.getTimestamp()); + long size = AbstractMemStore.heapSizeChange(kv, true); + regionServicesForStores.addAndGetGlobalMemstoreSize(size); + } + } + + private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge { + long t = 1234; + + @Override public long currentTime() { + return t; + } + + public void setCurrentTimeMillis(long t) { + this.t = t; + } + } + +} From 49ec8e13c2cb991b8dc7437faa1c98b6c41d8c95 Mon Sep 17 00:00:00 2001 From: anastas Date: Wed, 4 May 2016 10:34:21 +0300 Subject: [PATCH 8/9] Previous changes --- .../org/apache/hadoop/hbase/regionserver/CellArrayMap.java | 3 ++- .../org/apache/hadoop/hbase/regionserver/CellChunkMap.java | 2 ++ .../org/apache/hadoop/hbase/regionserver/CellFlatMap.java | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java index 5111ff9e45ce..ccf1f0794fa7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hbase.Cell; /** - * CellArrayMap is a simple array of Cells and can be allocated only using JVM. + * CellArrayMap is a simple array of Cells and can be allocated only using on-heap. * In contrast, CellChunkMap can be also allocated off-heap. * As all java arrays CellArrayMap's array of references pointing to Cell objects. */ @@ -47,6 +47,7 @@ protected CellFlatMap createSubCellFlatMap(Comparator comparator, @Override protected Cell getCell(int i) { + if(i=maxCellIdx) return null; return block[i]; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java index 4fa678ff9169..2880c4f16cf8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java @@ -79,6 +79,8 @@ protected CellFlatMap createSubCellFlatMap(Comparator comparator, @Override protected Cell getCell(int i) { + if(i=maxCellIdx) return null; + // find correct chunk int chunkIndex = (i / numOfCellsInsideChunk); byte[] block = chunks[chunkIndex].getData(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java index f50fbb995cfc..a3b589d431e3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java @@ -43,8 +43,8 @@ public abstract class CellFlatMap implements ConcurrentNavigableMap { private final Comparator comparator; - private int minCellIdx = 0; // the index of the minimal cell (for sub-sets) - private int maxCellIdx = 0; // the index of the maximal cell (for sub-sets) + protected int minCellIdx = 0; // the index of the minimal cell (for sub-sets) + protected int maxCellIdx = 0; // the index of the maximal cell (for sub-sets) private boolean descending = false; /* C-tor */ From e9b09bcc75182177218aa0b9b0e7b27297927143 Mon Sep 17 00:00:00 2001 From: anastas Date: Sun, 5 Jun 2016 10:52:30 +0300 Subject: [PATCH 9/9] Last changes on 14921 without master --- .../regionserver/CompactingMemStore.java | 20 +- .../hbase/regionserver/ImmutableSegment.java | 6 + .../TestCompactingToCellChunkMapMemStore.java | 236 ++++++++++++++++++ 3 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellChunkMapMemStore.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index 657fbe2e6391..04a4f91ba52b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -98,10 +98,19 @@ public CompactingMemStore(Configuration conf, CellComparator c, int t = conf.getInt(COMPACTING_MEMSTORE_TYPE_KEY, COMPACTING_MEMSTORE_TYPE_DEFAULT); switch (t) { case 1: type = Type.COMPACT_TO_SKIPLIST_MAP; + LOG.info("Creating CompactingMemStore that is going to compact to SkipList data structure:" + + " region " + getRegionServices().getRegionInfo() + .getRegionNameAsString() + " and store: "+ getFamilyName()); break; case 2: type = Type.COMPACT_TO_ARRAY_MAP; + LOG.info("Creating CompactingMemStore that is going to compact to CellArray data structure:" + + " region " + getRegionServices().getRegionInfo() + .getRegionNameAsString() + " and store: "+ getFamilyName()); break; case 3: type = Type.COMPACT_TO_CHUNK_MAP; + LOG.info("Creating CompactingMemStore that is going to compact to ChunkMap data structure:" + + " region " + getRegionServices().getRegionInfo() + .getRegionNameAsString() + " and store: "+ getFamilyName()); break; } } @@ -427,7 +436,7 @@ public boolean startCompact() throws IOException { // this local copy of the list is marked with specific version versionedList = pipeline.getVersionedList(); LOG.info( - "Starting the MemStore in-memory compaction for store " + store.getColumnFamilyName()); + "Starting the MemStore in-memory compaction for store: " + store.getColumnFamilyName()); doCompact(); return true; } @@ -467,7 +476,7 @@ private void doCompact() { > COMPACTION_TRIGGER_REMAIN_FACTOR * versionedList.getNumOfCells())) { // too much cells "survive" the possible compaction we do not want to compact! LOG.debug("In-Memory compaction does not pay off - storing the flattened segment" - + " for store " + getFamilyName()); + + " for store: " + getFamilyName()); // Looking for Segment in the pipeline with SkipList index, to make it flat pipeline.flattenYoungestSegment(versionedList.getVersion()); return; @@ -500,6 +509,10 @@ private void doCompact() { private ImmutableSegment compact(int numOfCells) throws IOException { + LOG.info( + "Starting in-memory compaction of type: " + type + ". Before compaction we have " + + numOfCells + " cells in the entire compaction pipeline"); + ImmutableSegment result = null; MemStoreCompactorIterator iterator = new MemStoreCompactorIterator(versionedList.getStoreSegments(), getComparator(), @@ -517,6 +530,9 @@ private ImmutableSegment compact(int numOfCells) ImmutableSegment.Type.ARRAY_MAP_BASED); break; case COMPACT_TO_CHUNK_MAP: +// org.junit.Assert.assertTrue("\n<<<< Compacting to CellChunkMap set for " + numOfCells +// + " cells. \n", false); + result = SegmentFactory.instance() .createImmutableSegment( getConfiguration(), getComparator(), iterator, numOfCells, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java index 3239bfd9c0b5..086b7d716338 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java @@ -90,6 +90,8 @@ protected ImmutableSegment( if (type == Type.ARRAY_MAP_BASED) { cs = createCellArrayMapSet(numOfCells, iterator); } else { +// org.junit.Assert.assertTrue("\n<<<< Creating CellChunkMap set for " + numOfCells +// + " cells. \n", false); cs = createCellChunkMapSet(numOfCells, iterator, conf); } this.setCellSet(null, cs); // update the CellSet of the new Segment @@ -223,6 +225,10 @@ private CellSet createCellChunkMapSet( chunks[currentChunkIdx] = ms.allocateChunk(); int offsetInCurentChunk = 0; + org.junit.Assert.assertTrue("\n<<<< Creating CellChunkMap set for " + numOfCells + + " cells. The calculated chunk size is " + chunkSize + " bytes. " + "We need " + + numberOfChunks + " chunks. " + "\n", false); + while (iterator.hasNext()) { Cell c = iterator.next(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellChunkMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellChunkMapMemStore.java new file mode 100644 index 000000000000..f3d7170c59a3 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellChunkMapMemStore.java @@ -0,0 +1,236 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.experimental.categories.Category; + +import java.io.IOException; + +/** + * compacted memstore test case + */ +@Category({RegionServerTests.class, MediumTests.class}) +public class TestCompactingToCellChunkMapMemStore extends TestCompactingMemStore { + + private static final Log LOG = LogFactory.getLog(TestCompactingToCellChunkMapMemStore.class); + //private static MemStoreChunkPool chunkPool; + //private HRegion region; + //private RegionServicesForStores regionServicesForStores; + //private HStore store; + + ////////////////////////////////////////////////////////////////////////////// + // Helpers + ////////////////////////////////////////////////////////////////////////////// + + @Override public void tearDown() throws Exception { + chunkPool.clearChunks(); + } + + @Override public void setUp() throws Exception { + compactingSetUp(); + this.memstore = + new CompactingMemStore(HBaseConfiguration.create(), CellComparator.COMPARATOR, store, + regionServicesForStores, CompactingMemStore.Type.COMPACT_TO_CHUNK_MAP); + } + + ////////////////////////////////////////////////////////////////////////////// + // Compaction tests + ////////////////////////////////////////////////////////////////////////////// + public void testCompaction1Bucket() throws IOException { + int counter = 0; + String[] keys1 = { "A", "A", "B", "C" }; //A1, A2, B3, C4 + + // test 1 bucket + addRowsByKeys(memstore, keys1); + assertEquals(704, regionServicesForStores.getGlobalMemstoreTotalSize()); + assertEquals(4, memstore.getActive().getCellsCount()); + long size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(440, regionServicesForStores.getGlobalMemstoreTotalSize()); + for ( Segment s : memstore.getListOfSegments()) { + counter += s.getCellsCount(); + } + assertEquals(3, counter); + size = memstore.getFlushableSize(); + MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot + region.addAndGetGlobalMemstoreSize(-size); // simulate flusher + ImmutableSegment s = memstore.getSnapshot(); + assertEquals(3, s.getCellsCount()); + assertEquals(0, regionServicesForStores.getGlobalMemstoreTotalSize()); + + memstore.clearSnapshot(snapshot.getId()); + } + + public void testCompaction2Buckets() throws IOException { + + String[] keys1 = { "A", "A", "B", "C" }; + String[] keys2 = { "A", "B", "D" }; + + addRowsByKeys(memstore, keys1); + assertEquals(704, regionServicesForStores.getGlobalMemstoreTotalSize()); + long size = memstore.getFlushableSize(); + +// assertTrue( +// "\n\n<<< This is the active size with 4 keys - " + memstore.getActive().getSize() +// + ". This is the memstore flushable size - " + size + "\n",false); + + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(1000); + } + int counter = 0; + for ( Segment s : memstore.getListOfSegments()) { + counter += s.getCellsCount(); + } + assertEquals(3,counter); + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(440, regionServicesForStores.getGlobalMemstoreTotalSize()); + + addRowsByKeys(memstore, keys2); + assertEquals(968, regionServicesForStores.getGlobalMemstoreTotalSize()); + + size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + int i = 0; + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + if (i > 10000000) { + ((CompactingMemStore) memstore).debug(); + assertTrue("\n\n<<< Infinite loop! :( \n", false); + } + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + counter = 0; + for ( Segment s : memstore.getListOfSegments()) { + counter += s.getCellsCount(); + } + assertEquals(4,counter); + assertEquals(592, regionServicesForStores.getGlobalMemstoreTotalSize()); + + size = memstore.getFlushableSize(); + MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot + region.addAndGetGlobalMemstoreSize(-size); // simulate flusher + ImmutableSegment s = memstore.getSnapshot(); + assertEquals(4, s.getCellsCount()); + assertEquals(0, regionServicesForStores.getGlobalMemstoreTotalSize()); + + memstore.clearSnapshot(snapshot.getId()); + } + + public void testCompaction3Buckets() throws IOException { + + String[] keys1 = { "A", "A", "B", "C" }; + String[] keys2 = { "A", "B", "D" }; + String[] keys3 = { "D", "B", "B" }; + + addRowsByKeys(memstore, keys1); + assertEquals(704, region.getMemstoreSize()); + + long size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + + String tstStr = "\n\nFlushable size after first flush in memory:" + size + ". Is MemmStore in compaction?:" + + ((CompactingMemStore) memstore).isMemStoreFlushingInMemory(); + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(440, regionServicesForStores.getGlobalMemstoreTotalSize()); + + addRowsByKeys(memstore, keys2); + + tstStr += " After adding second part of the keys. Memstore size: " + + region.getMemstoreSize() + ", Memstore Total Size: " + + regionServicesForStores.getGlobalMemstoreTotalSize() + "\n\n"; + + assertEquals(968, regionServicesForStores.getGlobalMemstoreTotalSize()); + + ((CompactingMemStore) memstore).disableCompaction(); + size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline without compaction + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(968, regionServicesForStores.getGlobalMemstoreTotalSize()); + + addRowsByKeys(memstore, keys3); + assertEquals(1496, regionServicesForStores.getGlobalMemstoreTotalSize()); + + ((CompactingMemStore) memstore).enableCompaction(); + size = memstore.getFlushableSize(); + ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + assertEquals(592, regionServicesForStores.getGlobalMemstoreTotalSize()); + + size = memstore.getFlushableSize(); + MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot + region.addAndGetGlobalMemstoreSize(-size); // simulate flusher + ImmutableSegment s = memstore.getSnapshot(); + assertEquals(4, s.getCellsCount()); + assertEquals(0, regionServicesForStores.getGlobalMemstoreTotalSize()); + + memstore.clearSnapshot(snapshot.getId()); + + //assertTrue(tstStr, false); + } + + private void addRowsByKeys(final AbstractMemStore hmc, String[] keys) { + byte[] fam = Bytes.toBytes("testfamily"); + byte[] qf = Bytes.toBytes("testqualifier"); + for (int i = 0; i < keys.length; i++) { + long timestamp = System.currentTimeMillis(); + Threads.sleep(1); // to make sure each kv gets a different ts + byte[] row = Bytes.toBytes(keys[i]); + byte[] val = Bytes.toBytes(keys[i] + i); + KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); + hmc.add(kv); + LOG.debug("added kv: " + kv.getKeyString() + ", timestamp" + kv.getTimestamp()); + long size = AbstractMemStore.heapSizeChange(kv, true); + regionServicesForStores.addAndGetGlobalMemstoreSize(size); + } + } + + private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge { + long t = 1234; + + @Override public long currentTime() { + return t; + } + + public void setCurrentTimeMillis(long t) { + this.t = t; + } + } + +}