diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlock.java new file mode 100644 index 000000000000..6403a5266f1f --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlock.java @@ -0,0 +1,467 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Cellersion 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY CellIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.Cell; + +import java.util.Collection; +import java.util.Comparator; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Set; + + + +/** + * CellBlock stores a constant number of elements and is immutable after creation stage. + * Due to being immutable the CellBlock can be implemented as array. + * The actual array is on- or off-heap and is implemented in concrete class derived from CellBlock. + * The CellBlock uses no synchronization primitives, it is assumed to be created by a + * single thread and then it can be read-only by multiple threads. + */ +public abstract class CellBlock implements ConcurrentNavigableMap { + + private final Comparator comparator; + private int minCellIdx = 0; // the index of the minimal cell (for sub-sets) + private int maxCellIdx = 0; // the index of the maximal cell (for sub-sets) + private boolean descending = false; + + /* C-tor */ + public CellBlock(Comparator comparator, int min, int max, boolean d){ + this.comparator = comparator; + this.minCellIdx = min; + this.maxCellIdx = max; + this.descending = d; + } + + /* Used for abstract CellBlock creation, implemented by derived class */ + protected abstract CellBlock createCellBlocks(Comparator comparator, int min, + int max, boolean descending); + + /* Returns the i-th cell in the cell block */ + protected abstract Cell getCellFromIndex(int i); + + /** + * Binary search for a given key in between given boundaries of the array. + * Positive returned numbers mean the index. + * Negative returned numbers means the key not found. + * The absolute value of the output is the + * possible insert index for the searched key: (-1 * insertion point) - 1 + * @param needle The key to look for in all of the entries + * @return Same return value as Arrays.binarySearch. + */ + private int find(Cell needle) { + int begin = minCellIdx; + int end = maxCellIdx - 1; + + while (begin <= end) { + int mid = begin + ((end - begin) / 2); + + Cell midCell = getCellFromIndex(mid); + + int compareRes = comparator.compare(midCell, needle); + + // 0 means equals. We found the key. + if (compareRes == 0) return mid; + else if (compareRes < 0) { + // midCell is less than needle so we need to look at farther up + begin = mid + 1; + } else { + // midCell is greater than needle so we need to look down + end = mid - 1; + } + } + + return (-1 * begin) - 1; + } + + private int getValidIndex(Cell key, boolean inclusive) { + int index = find(key); + if (inclusive && index >= 0) index++; + else if (index < 0) index = -(index + 1) - 1; + return index; + } + + @Override + public Comparator comparator() { + return comparator; + } + + @Override + public int size() { + return maxCellIdx-minCellIdx; + } + + @Override + public boolean isEmpty() { + return (maxCellIdx==minCellIdx); + } + + + // ---------------- Sub-Maps ---------------- + @Override + public ConcurrentNavigableMap subMap( Cell fromKey, + boolean fromInclusive, + Cell toKey, + boolean toInclusive) { + int toIndex = getValidIndex(toKey, toInclusive); + int fromIndex = (getValidIndex(fromKey, !fromInclusive)); + + if (fromIndex > toIndex) throw new IllegalArgumentException("inconsistent range"); + return createCellBlocks(comparator, fromIndex, toIndex, descending); + } + + @Override + public ConcurrentNavigableMap headMap(Cell toKey, boolean inclusive) { + int index = getValidIndex(toKey, inclusive); + return createCellBlocks(comparator, minCellIdx, index, descending); + } + + @Override + public ConcurrentNavigableMap tailMap(Cell fromKey, boolean inclusive) { + int index = (getValidIndex(fromKey, !inclusive)); + return createCellBlocks(comparator, index, maxCellIdx, descending); + } + + @Override + public ConcurrentNavigableMap descendingMap() { + return createCellBlocks(comparator, minCellIdx, maxCellIdx, true); + } + + @Override + public ConcurrentNavigableMap subMap(Cell k1, Cell k2) { + return this.subMap(k1, true, k2, true); + } + + @Override + public ConcurrentNavigableMap headMap(Cell k) { + return this.headMap(k, true); + } + + @Override + public ConcurrentNavigableMap tailMap(Cell k) { + return this.tailMap(k, true); + } + + + // -------------------------------- Key's getters -------------------------------- + @Override + public Cell firstKey() { + if (isEmpty()) return null; + if (descending) getCellFromIndex(maxCellIdx-1); + return getCellFromIndex(minCellIdx); + } + + @Override + public Cell lastKey() { + if (isEmpty()) return null; + if (descending) return getCellFromIndex(minCellIdx); + return getCellFromIndex(maxCellIdx-1); + } + + @Override + public Cell lowerKey(Cell k) { + if (isEmpty()) return null; + int index = find(k); + if (descending) { + if (index >= 0) index++; // There's a key exactly equal. + else index = -(index + 1); + } else { + if (index >= 0) index--; // There's a key exactly equal. + else index = -(index + 1) - 1; + } + return (index < minCellIdx || index >= maxCellIdx) ? null : getCellFromIndex(index); + } + + @Override + public Cell floorKey(Cell k) { + if (isEmpty()) return null; + int index = find(k); + if (descending) { + if (index < 0) index = -(index + 1); + } else { + if (index < 0) index = -(index + 1) - 1; + } + return (index < minCellIdx || index >= maxCellIdx) ? null : getCellFromIndex(index); + } + + @Override + public Cell ceilingKey(Cell k) { + if (isEmpty()) return null; + int index = find(k); + if (descending) { + if (index < 0) index = -(index + 1) - 1; + } else { + if (index < 0) index = -(index + 1); + } + return (index < minCellIdx || index >= maxCellIdx) ? null : getCellFromIndex(index); + } + + @Override + public Cell higherKey(Cell k) { + if (isEmpty()) return null; + int index = find(k); + if (descending) { + if (index >= 0) index--; // There's a key exactly equal. + else index = -(index + 1) - 1; + } else { + if (index >= 0) index++; // There's a key exactly equal. + else index = -(index + 1); + } + return (index < minCellIdx || index >= maxCellIdx) ? null : getCellFromIndex(index); + } + + @Override + public boolean containsKey(Object o) { + int index = find((Cell) o); + return (index >= 0); + } + + @Override + public boolean containsValue(Object o) { // use containsKey(Object o) instead + throw new UnsupportedOperationException(); + } + + @Override + public Cell get(Object o) { + int index = find((Cell) o); + if (index >= 0) { + return getCellFromIndex(index); + } + return null; + } + + // -------------------------------- Entry's getters -------------------------------- + // all interfaces returning Entries are unsupported because we are dealing only with the keys + @Override + public Entry lowerEntry(Cell k) { + throw new UnsupportedOperationException(); + } + + @Override + public Entry higherEntry(Cell k) { + throw new UnsupportedOperationException(); + } + + @Override + public Entry ceilingEntry(Cell k) { + throw new UnsupportedOperationException(); + } + + @Override + public Entry floorEntry(Cell k) { + throw new UnsupportedOperationException(); + } + + @Override + public Entry firstEntry() { + throw new UnsupportedOperationException(); + } + + @Override + public Entry lastEntry() { + throw new UnsupportedOperationException(); + } + + @Override + public Entry pollFirstEntry() { + throw new UnsupportedOperationException(); + } + + @Override + public Entry pollLastEntry() { + throw new UnsupportedOperationException(); + } + + + // -------------------------------- Updates -------------------------------- + // All updating methods below are unsupported. + // Assuming an array of Cells will be allocated externally, + // fill up with Cells and provided in construction time. + // Later the structure is immutable. + @Override + public Cell put(Cell k, Cell v) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + @Override + public Cell remove(Object o) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean replace(Cell k, Cell v, Cell v1) { + throw new UnsupportedOperationException(); + } + + @Override + public void putAll(Map map) { + throw new UnsupportedOperationException(); + } + + @Override + public Cell putIfAbsent(Cell k, Cell v) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean remove(Object o, Object o1) { + throw new UnsupportedOperationException(); + } + + @Override + public Cell replace(Cell k, Cell v) { + throw new UnsupportedOperationException(); + } + + + // -------------------------------- Sub-Sets -------------------------------- + @Override + public NavigableSet navigableKeySet() { + throw new UnsupportedOperationException(); + } + + @Override + public NavigableSet descendingKeySet() { + throw new UnsupportedOperationException(); + } + + @Override + public NavigableSet keySet() { + throw new UnsupportedOperationException(); + } + + @Override + public Collection values() { + return new CellBlocksCollection(); + } + + @Override + public Set> entrySet() { + throw new UnsupportedOperationException(); + } + + + // -------------------------------- Iterator K -------------------------------- + private final class CellBlocksIterator implements Iterator { + int index; + + private CellBlocksIterator() { + index = descending ? maxCellIdx-1 : minCellIdx; + } + + @Override + public boolean hasNext() { + return descending ? (index >= minCellIdx) : (index < maxCellIdx); + } + + @Override + public Cell next() { + Cell result = getCellFromIndex(index); + if (descending) index--; + else index++; + return result; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } + + + // -------------------------------- Collection -------------------------------- + private final class CellBlocksCollection implements Collection { + + @Override + public int size() { + return CellBlock.this.size(); + } + + @Override + public boolean isEmpty() { + return CellBlock.this.isEmpty(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean contains(Object o) { + return containsKey(o); + } + + @Override + public Iterator iterator() { + return new CellBlocksIterator(); + } + + @Override + public Object[] toArray() { + throw new UnsupportedOperationException(); + } + + @Override + public T[] toArray(T[] ts) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean add(Cell k) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean remove(Object o) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean containsAll(Collection collection) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean addAll(Collection collection) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean removeAll(Collection collection) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean retainAll(Collection collection) { + throw new UnsupportedOperationException(); + } + + + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockObjectArray.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockObjectArray.java new file mode 100644 index 000000000000..bebdd7aee5f7 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockObjectArray.java @@ -0,0 +1,52 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Cellersion 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY CellIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.util.Comparator; +import org.apache.hadoop.hbase.Cell; + +/** + * CellBlockObjectArray is a simple array of Cells allocated using JVM. + * As all java arrays it is array of references pointing to Cell objects + */ +public class CellBlockObjectArray extends CellBlock { + + Cell[] block; + + /* The Cells Array is created only when CellBlockObjectArray is created, all sub-CellBlocks use + * boundary indexes */ + public CellBlockObjectArray(Comparator comparator, Cell[] b, int min, int max, + boolean d) { + super(comparator,min,max,d); + this.block = b; + } + + /* To be used by base class only to create a sub-CellBlock */ + @Override + protected CellBlock createCellBlocks(Comparator comparator, + int min, int max, boolean d) { + return new CellBlockObjectArray(comparator,this.block,min,max,d); + } + + @Override + protected Cell getCellFromIndex(int i) { + return block[i]; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockSerialized.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockSerialized.java new file mode 100644 index 000000000000..003cb952db66 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellBlockSerialized.java @@ -0,0 +1,85 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Cellersion 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY CellIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.util.Comparator; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + + + +/** + * CellBlockSerialized is a byte array holding all that is needed to access a Cell, which + * is actually saved on another deeper byte array. + * Per Cell we have a reference to this deeper byte array B, offset in bytes in B (integer), + * and length in bytes in B (integer). In order to save reference to byte array we use the Chunk's + * indexes given by MSLAB (also integer). + * + * The B memory layout: + * + * <----------------- first Cell ---------------------> <-------------- second Cell + * ------------------------------------------------------------------------------------- ... + * | integer = x bytes | integer = x bytes | integer = x bytes | integer = x bytes | + * | reference to B | offset in B where | length of Cell's | reference to may be| ... + * | holding Cell data | Cell's data starts| data in B | another byte array | + * ------------------------------------------------------------------------------------- ... + */ +public class CellBlockSerialized extends CellBlock { + + private HeapMemStoreLAB.Chunk[] chunks; + private final HeapMemStoreLAB memStoreLAB; + private int numOfCellsInsideChunk; + private static final int BYTES_IN_CELL = 3*(Integer.SIZE / Byte.SIZE); // each Cell requires 3 integers + + public CellBlockSerialized(Comparator comparator, HeapMemStoreLAB memStoreLAB, + HeapMemStoreLAB.Chunk[] chunks, int min, int max, int chunkSize, boolean d) { + super(comparator,min,max, d); + this.chunks = chunks; + this.memStoreLAB = memStoreLAB; + this.numOfCellsInsideChunk = chunkSize / BYTES_IN_CELL; + } + + /* To be used by base class only to create a sub-CellBlock */ + @Override + protected CellBlock createCellBlocks(Comparator comparator, + int min, int max, boolean d) { + return new CellBlockSerialized(comparator, this.memStoreLAB, this.chunks, min, max, + this.numOfCellsInsideChunk* BYTES_IN_CELL, d); + } + + @Override + protected Cell getCellFromIndex(int i) { + // find correct chunk + int chunkIndex = (i / numOfCellsInsideChunk); + byte[] block = chunks[chunkIndex].getData(); + i = i - chunkIndex*numOfCellsInsideChunk; + + // find inside chunk + int offsetInBytes = i* BYTES_IN_CELL; + int chunkId = Bytes.toInt(block,offsetInBytes); + int offsetOfCell = Bytes.toInt(block,offsetInBytes+(Integer.SIZE / Byte.SIZE)); + int lengthOfCell = Bytes.toInt(block,offsetInBytes+2*(Integer.SIZE / Byte.SIZE)); + byte[] chunk = memStoreLAB.translateIdToChunk(chunkId).getData(); + + Cell result = new KeyValue(chunk, offsetOfCell, lengthOfCell); + return result; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java index f22a6e5c1ac8..228616bade7e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java @@ -31,6 +31,7 @@ import com.google.common.base.Preconditions; + /** * A memstore-local allocation buffer. *

@@ -88,9 +89,8 @@ public HeapMemStoreLAB(Configuration conf) { this.chunkPool = MemStoreChunkPool.getPool(conf); // if we don't exclude allocations >CHUNK_SIZE, we'd infiniteloop on one! - Preconditions.checkArgument( - maxAlloc <= chunkSize, - MAX_ALLOC_KEY + " must be less than " + CHUNK_SIZE_KEY); + Preconditions.checkArgument(maxAlloc <= chunkSize, + MAX_ALLOC_KEY + " must be less than " + CHUNK_SIZE_KEY); } /** @@ -112,7 +112,7 @@ public ByteRange allocateBytes(int size) { while (true) { Chunk c = getOrMakeChunk(); - // Try to allocate from this chunk + // Try to allocate from this chunk int allocOffset = c.alloc(size); if (allocOffset != -1) { // We succeeded - this is the common case - small alloc @@ -181,6 +181,7 @@ private void tryRetireChunk(Chunk c) { * allocate a new one from the JVM. */ private Chunk getOrMakeChunk() { + while (true) { // Try to get the chunk Chunk c = curChunk.get(); @@ -191,11 +192,21 @@ private Chunk getOrMakeChunk() { // No current chunk, so we want to allocate one. We race // against other allocators to CAS in an uninitialized chunk // (which is cheap to allocate) - c = (chunkPool != null) ? chunkPool.getChunk() : new Chunk(chunkSize); + + //c = (chunkPool != null) ? chunkPool.getChunk() : new Chunk(chunkSize, 5); //14921 + + if(chunkPool != null) { + c = chunkPool.getChunk(); + } else { + c = new Chunk(chunkSize, 5); + c.init(); + } + if (curChunk.compareAndSet(null, c)) { // we won race - now we need to actually do the expensive // allocation step - c.init(); + + //c.init(); //14921 this.chunkQueue.add(c); return c; } else if (chunkPool != null) { @@ -206,6 +217,24 @@ private Chunk getOrMakeChunk() { } } + /** 14921 + * Given a chunk ID return reference to the relevant chunk + * @return a chunk + */ + Chunk translateIdToChunk(int id) { + return chunkPool.translateIdToChunk(id); + } + + /** 14921 + * Use instead of allocateBytes() when new full chunk is needed + * @return a chunk + */ + Chunk allocateChunk() { + Chunk c = chunkPool.getChunk(); + this.chunkQueue.add(c); + return c; + } + /** * A chunk of memory out of which allocations are sliced. */ @@ -227,12 +256,18 @@ static class Chunk { /** Size of chunk in bytes */ private final int size; + /* 14921: A unique identifier of a chunk inside MemStoreChunkPool */ + private final int id; + + /* Chunk's index serves as replacement for pointer */ + /** * Create an uninitialized chunk. Note that memory is not allocated yet, so * this is cheap. * @param size in bytes */ - Chunk(int size) { + Chunk(int size, int id) { + this.id = id; this.size = size; } @@ -252,13 +287,15 @@ public void init() { assert failInit; // should be true. throw e; } + // Mark that it's ready for use boolean initted = nextFreeOffset.compareAndSet( UNINITIALIZED, 0); // We should always succeed the above CAS since only one thread // calls init()! - Preconditions.checkState(initted, - "Multiple threads tried to init same chunk"); + Preconditions.checkState(initted, "Multiple threads tried to init same chunk"); + + //org.junit.Assert.assertTrue("\n\n inside chunk initialization 3", false); } /** @@ -311,5 +348,13 @@ public String toString() { " allocs=" + allocCount.get() + "waste=" + (data.length - nextFreeOffset.get()); } + + public int getId() { + return id; + } // 14921 + + public byte[] getData() { + return data; + } // 14921 } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java index 628506059eef..bc85bc0e95ee 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java @@ -19,12 +19,16 @@ package org.apache.hadoop.hbase.regionserver; import java.lang.management.ManagementFactory; +//import java.util.concurrent.*; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -76,17 +80,21 @@ public class MemStoreChunkPool { private static final int statThreadPeriod = 60 * 5; private AtomicLong createdChunkCount = new AtomicLong(); private AtomicLong reusedChunkCount = new AtomicLong(); + private AtomicInteger chunkIDs = new AtomicInteger(1); // 14921 + + // 14921: IDs Mapping of all chunks (key 0 is forbidden) + private final ConcurrentMap chunksMap = new ConcurrentHashMap(); MemStoreChunkPool(Configuration conf, int chunkSize, int maxCount, int initialCount) { this.maxCount = maxCount; this.chunkSize = chunkSize; this.reclaimedChunks = new LinkedBlockingQueue(); + for (int i = 0; i < initialCount; i++) { - Chunk chunk = new Chunk(chunkSize); - chunk.init(); - reclaimedChunks.add(chunk); + Chunk chunk = allocateChunk(); } + final String n = Thread.currentThread().getName(); scheduleThreadPool = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat(n+"-MemStoreChunkPool Statistics") @@ -103,10 +111,11 @@ public class MemStoreChunkPool { Chunk getChunk() { Chunk chunk = reclaimedChunks.poll(); if (chunk == null) { - chunk = new Chunk(chunkSize); + chunk = allocateChunk(); createdChunkCount.incrementAndGet(); } else { chunk.reset(); + chunk.init(); // 14921 reusedChunkCount.incrementAndGet(); } return chunk; @@ -125,6 +134,14 @@ void putbackChunks(BlockingQueue chunks) { chunks.drainTo(reclaimedChunks, maxNumToPutback); } + /** + * Given a chunk ID return reference to the relevant chunk + * @return a chunk + */ + Chunk translateIdToChunk(int id) { + return chunksMap.get(id); + } + /** * Add the chunk to the pool, if the pool has achieved the max size, it will * skip it @@ -148,6 +165,24 @@ void clearChunks() { this.reclaimedChunks.clear(); } + /* + * Only used in testing + */ + ConcurrentMap getChunksMap() { + return this.chunksMap; + } + + /* + * Allocate and register Chunk + */ + private Chunk allocateChunk() { + int newId = chunkIDs.getAndAdd(1); // the number of the new chunk + Chunk chunk = new Chunk(chunkSize,newId); + chunksMap.put(newId, chunk); + chunk.init(); + return chunk; + } + private static class StatisticsThread extends Thread { MemStoreChunkPool mcp; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java new file mode 100644 index 000000000000..1a368df9ea96 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellBlocksSet.java @@ -0,0 +1,174 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import junit.framework.TestCase; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +import java.util.Iterator; +import java.util.SortedSet; +import static org.junit.Assert.assertTrue; + +@Category({RegionServerTests.class, SmallTests.class}) +public class TestCellBlocksSet extends TestCase { + + private static final int NUM_OF_CELLS = 3; + + private Cell cells[]; + private CellBlockObjectArray cbOnHeap; + private CellBlockSerialized cbOffHeap; + + private final static Configuration conf = new Configuration(); + private HeapMemStoreLAB mslab; + + + + protected void setUp() throws Exception { + super.setUp(); + + // create array of Cells to bass to the CellBlock under CellSet + final byte[] one = Bytes.toBytes(1); + final byte[] two = Bytes.toBytes(2); + final byte[] three = Bytes.toBytes(3); + final byte[] f = Bytes.toBytes("f"); + final byte[] q = Bytes.toBytes("q"); + final byte[] v = Bytes.toBytes(4); + + final KeyValue kv1 = new KeyValue(one, f, q, 10, v); + final KeyValue kv2 = new KeyValue(two, f, q, 20, v); + final KeyValue kv3 = new KeyValue(three, f, q, 30, v); + + cells = new Cell[] {kv1,kv2,kv3}; + cbOnHeap = new CellBlockObjectArray(CellComparator.COMPARATOR,cells,0,NUM_OF_CELLS,false); + + conf.setBoolean(SegmentFactory.USEMSLAB_KEY, true); + conf.setFloat(MemStoreChunkPool.CHUNK_POOL_MAXSIZE_KEY, 0.2f); + MemStoreChunkPool.chunkPoolDisabled = false; + mslab = new HeapMemStoreLAB(conf); + + HeapMemStoreLAB.Chunk[] c = shallowCellsToBuffer(kv1, kv2, kv3); + int chunkSize = conf.getInt(HeapMemStoreLAB.CHUNK_SIZE_KEY, HeapMemStoreLAB.CHUNK_SIZE_DEFAULT); + cbOffHeap = new CellBlockSerialized(CellComparator.COMPARATOR, mslab, + c, 0, NUM_OF_CELLS, chunkSize, false); + } + + /* Create and test CellSet based on CellBlockObjectArray */ + public void testCellBlocksOnHeap() throws Exception { + CellSet cs = new CellSet(cbOnHeap); + testCellBlocks(cs); + testIterators(cs); + } + + /* Create and test CellSet based on CellBlockSerialized */ + public void testCellBlocksOffHeap() throws Exception { + CellSet cs = new CellSet(cbOffHeap); + testCellBlocks(cs); + testIterators(cs); + } + + /* Generic basic test for immutable CellSet */ + private void testCellBlocks(CellSet cs) throws Exception { + assertEquals(NUM_OF_CELLS, cs.size()); // check size + + assertTrue(cs.contains(cells[0])); // check existance of the first + Cell first = cs.first(); + assertTrue(cells[0].equals(first)); + + assertTrue(cs.contains(cells[NUM_OF_CELLS - 1])); // check last + Cell last = cs.last(); + assertTrue(cells[NUM_OF_CELLS - 1].equals(last)); + + SortedSet tail = cs.tailSet(cells[1]); // check tail abd head sizes + assertEquals(2, tail.size()); + SortedSet head = cs.headSet(cells[1]); + assertEquals(1, head.size()); + + Cell tailFirst = tail.first(); + assertTrue(cells[1].equals(tailFirst)); + Cell tailLast = tail.last(); + assertTrue(cells[2].equals(tailLast)); + + Cell headFirst = head.first(); + assertTrue(cells[0].equals(headFirst)); + Cell headLast = head.last(); + assertTrue(cells[0].equals(headLast)); + } + + /* Generic iterators test for immutable CellSet */ + private void testIterators(CellSet cs) throws Exception { + + // Assert that we have NUM_OF_CELLS values and that they are in order + int count = 0; + for (Cell kv: cs) { + assertEquals("\n\n-------------------------------------------------------------------\n" + + "Comparing iteration number " + (count + 1) + " the returned cell: " + kv + + ", the first Cell in the CellBlocksMap: " + cells[count] + + ", and the same transformed to String: " + cells[count].toString() + + "\n-------------------------------------------------------------------\n", + cells[count], kv); + count++; + } + assertEquals(NUM_OF_CELLS, count); + + // Test descending iterator + count = 0; + for (Iterator i = cs.descendingIterator(); i.hasNext();) { + Cell kv = i.next(); + assertEquals(cells[NUM_OF_CELLS - (count + 1)], kv); + count++; + } + assertEquals(NUM_OF_CELLS, count); + } + + /* Create byte array holding shallow Cells referencing to the deep Cells data */ + private HeapMemStoreLAB.Chunk[] shallowCellsToBuffer(Cell kv1, Cell kv2, Cell kv3) { + HeapMemStoreLAB.Chunk chunkD = mslab.allocateChunk(); + HeapMemStoreLAB.Chunk chunkS = mslab.allocateChunk(); + HeapMemStoreLAB.Chunk result[] = {chunkS}; + + byte[] deepBuffer = chunkD.getData(); + byte[] shallowBuffer = chunkS.getData(); + int offset = 0; + int pos = offset; + KeyValueUtil.appendToByteArray(kv1, deepBuffer, offset); // write deep cell data + + pos = Bytes.putInt(shallowBuffer, pos, chunkD.getId()); // write deep chunk index + pos = Bytes.putInt(shallowBuffer, pos, offset); // offset + pos = Bytes.putInt(shallowBuffer, pos, KeyValueUtil.length(kv1)); // length + offset += KeyValueUtil.length(kv1); + + KeyValueUtil.appendToByteArray(kv2, deepBuffer, offset); // write deep cell data + pos = Bytes.putInt(shallowBuffer, pos, chunkD.getId()); // deep chunk index + pos = Bytes.putInt(shallowBuffer, pos, offset); // offset + pos = Bytes.putInt(shallowBuffer, pos, KeyValueUtil.length(kv2)); // length + offset += KeyValueUtil.length(kv2); + + KeyValueUtil.appendToByteArray(kv3, deepBuffer, offset); // write deep cell data + pos = Bytes.putInt(shallowBuffer, pos, chunkD.getId()); // deep chunk index + pos = Bytes.putInt(shallowBuffer, pos, offset); // offset + pos = Bytes.putInt(shallowBuffer, pos, KeyValueUtil.length(kv3)); // length + + return result; + } +}