Skip to content

Commit f3050cf

Browse files
szetszwoszetszwo
authored
HDDS-13276. Use KEY_ONLY/VALUE_ONLY iterator in SCM/Datanode. (apache#8638)
1 parent e9c0a45 commit f3050cf

File tree

17 files changed

+58
-97
lines changed

17 files changed

+58
-97
lines changed

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.hadoop.ozone.container.keyvalue;
1919

20+
import static org.apache.hadoop.hdds.utils.db.Table.KeyValueIterator.Type.VALUE_ONLY;
2021
import static org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil.isSameSchemaVersion;
2122

2223
import com.fasterxml.jackson.databind.JsonNode;
@@ -35,7 +36,6 @@
3536
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
3637
import org.apache.hadoop.hdds.server.JsonUtils;
3738
import org.apache.hadoop.hdds.utils.db.Table;
38-
import org.apache.hadoop.hdds.utils.db.TableIterator;
3939
import org.apache.hadoop.ozone.OzoneConsts;
4040
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
4141
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@@ -500,8 +500,7 @@ static PendingDelete countPendingDeletesSchemaV2(
500500
Table<Long, DeletedBlocksTransaction> delTxTable =
501501
schemaTwoStore.getDeleteTransactionTable();
502502

503-
try (TableIterator<Long, ? extends Table.KeyValue<Long,
504-
DeletedBlocksTransaction>> iterator = delTxTable.iterator()) {
503+
try (Table.KeyValueIterator<Long, DeletedBlocksTransaction> iterator = delTxTable.iterator(VALUE_ONLY)) {
505504
while (iterator.hasNext()) {
506505
DeletedBlocksTransaction txn = iterator.next().getValue();
507506
final List<Long> localIDs = txn.getLocalIDList();
@@ -544,11 +543,8 @@ static PendingDelete countPendingDeletesSchemaV3(
544543
KeyValueContainerData containerData) throws IOException {
545544
long pendingDeleteBlockCountTotal = 0;
546545
long pendingDeleteBytes = 0;
547-
try (
548-
TableIterator<String, ? extends Table.KeyValue<String,
549-
DeletedBlocksTransaction>>
550-
iter = store.getDeleteTransactionTable()
551-
.iterator(containerData.containerPrefix())) {
546+
try (Table.KeyValueIterator<String, DeletedBlocksTransaction> iter
547+
= store.getDeleteTransactionTable().iterator(containerData.containerPrefix(), VALUE_ONLY)) {
552548
while (iter.hasNext()) {
553549
DeletedBlocksTransaction delTx = iter.next().getValue();
554550
final List<Long> localIDs = delTx.getLocalIDList();

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingTask.java‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.hadoop.ozone.container.keyvalue.statemachine.background;
1919

20+
import static org.apache.hadoop.hdds.utils.db.Table.KeyValueIterator.Type.VALUE_ONLY;
2021
import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V1;
2122
import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V2;
2223
import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V3;
@@ -39,7 +40,6 @@
3940
import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
4041
import org.apache.hadoop.hdds.utils.db.BatchOperation;
4142
import org.apache.hadoop.hdds.utils.db.Table;
42-
import org.apache.hadoop.hdds.utils.db.TableIterator;
4343
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
4444
import org.apache.hadoop.ozone.container.common.helpers.BlockDeletingServiceMetrics;
4545
import org.apache.hadoop.ozone.container.common.impl.BlockDeletingService;
@@ -277,9 +277,7 @@ public ContainerBackgroundTaskResult deleteViaSchema2(
277277
Table<Long, DeletedBlocksTransaction> deleteTxns =
278278
((DeleteTransactionStore<Long>) meta.getStore())
279279
.getDeleteTransactionTable();
280-
try (TableIterator<Long,
281-
? extends Table.KeyValue<Long, DeletedBlocksTransaction>>
282-
iterator = deleteTxns.iterator()) {
280+
try (Table.KeyValueIterator<Long, DeletedBlocksTransaction> iterator = deleteTxns.iterator(VALUE_ONLY)) {
283281
return deleteViaTransactionStore(
284282
iterator, meta,
285283
container, dataDir, startTime, schema2Deleter);
@@ -298,18 +296,16 @@ public ContainerBackgroundTaskResult deleteViaSchema3(
298296
Table<String, DeletedBlocksTransaction> deleteTxns =
299297
((DeleteTransactionStore<String>) meta.getStore())
300298
.getDeleteTransactionTable();
301-
try (TableIterator<String,
302-
? extends Table.KeyValue<String, DeletedBlocksTransaction>>
303-
iterator = deleteTxns.iterator(containerData.containerPrefix())) {
299+
try (Table.KeyValueIterator<String, DeletedBlocksTransaction> iterator
300+
= deleteTxns.iterator(containerData.containerPrefix(), VALUE_ONLY)) {
304301
return deleteViaTransactionStore(
305302
iterator, meta,
306303
container, dataDir, startTime, schema3Deleter);
307304
}
308305
}
309306

310307
private ContainerBackgroundTaskResult deleteViaTransactionStore(
311-
TableIterator<?, ? extends Table.KeyValue<?, DeletedBlocksTransaction>>
312-
iter, DBHandle meta, Container container, File dataDir,
308+
Table.KeyValueIterator<?, DeletedBlocksTransaction> iter, DBHandle meta, Container container, File dataDir,
313309
long startTime, Deleter deleter) throws IOException {
314310
ContainerBackgroundTaskResult crr = new ContainerBackgroundTaskResult();
315311
if (!checkDataDir(dataDir)) {

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractDatanodeStore.java‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
import org.apache.hadoop.hdds.utils.db.DBStore;
3131
import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
3232
import org.apache.hadoop.hdds.utils.db.Table;
33-
import org.apache.hadoop.hdds.utils.db.TableIterator;
3433
import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions;
3534
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
3635
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfoList;
@@ -208,9 +207,7 @@ public static class KeyValueBlockIterator implements
208207
private static final Logger LOG = LoggerFactory.getLogger(
209208
KeyValueBlockIterator.class);
210209

211-
private final TableIterator<String, ? extends Table.KeyValue<String,
212-
BlockData>>
213-
blockIterator;
210+
private final Table.KeyValueIterator<String, BlockData> blockIterator;
214211
private static final KeyPrefixFilter DEFAULT_BLOCK_FILTER =
215212
MetadataKeyFilters.getUnprefixedKeyFilter();
216213
private final KeyPrefixFilter blockFilter;
@@ -221,9 +218,7 @@ public static class KeyValueBlockIterator implements
221218
* KeyValueBlockIterator to iterate unprefixed blocks in a container.
222219
* @param iterator - The underlying iterator to apply the block filter to.
223220
*/
224-
KeyValueBlockIterator(long containerID,
225-
TableIterator<String, ? extends Table.KeyValue<String, BlockData>>
226-
iterator) {
221+
KeyValueBlockIterator(long containerID, Table.KeyValueIterator<String, BlockData> iterator) {
227222
this.containerID = containerID;
228223
this.blockIterator = iterator;
229224
this.blockFilter = DEFAULT_BLOCK_FILTER;
@@ -235,8 +230,7 @@ public static class KeyValueBlockIterator implements
235230
* @param filter - Block filter, filter to be applied for blocks
236231
*/
237232
KeyValueBlockIterator(long containerID,
238-
TableIterator<String, ? extends Table.KeyValue<String, BlockData>>
239-
iterator, KeyPrefixFilter filter) {
233+
Table.KeyValueIterator<String, BlockData> iterator, KeyPrefixFilter filter) {
240234
this.containerID = containerID;
241235
this.blockIterator = iterator;
242236
this.blockFilter = filter;
@@ -313,8 +307,7 @@ public static class KeyValueBlockLocalIdIterator implements
313307
private static final Logger LOG = LoggerFactory.getLogger(
314308
KeyValueBlockLocalIdIterator.class);
315309

316-
private final TableIterator<String, ? extends Table.KeyValue<String,
317-
Long>> blockLocalIdIterator;
310+
private final Table.KeyValueIterator<String, Long> blockLocalIdIterator;
318311
private final KeyPrefixFilter localIdFilter;
319312
private Long nextLocalId;
320313
private final long containerID;
@@ -325,8 +318,7 @@ public static class KeyValueBlockLocalIdIterator implements
325318
* @param filter - BlockLocalId filter to be applied for block localIds.
326319
*/
327320
KeyValueBlockLocalIdIterator(long containerID,
328-
TableIterator<String, ? extends Table.KeyValue<String, Long>>
329-
iterator, KeyPrefixFilter filter) {
321+
Table.KeyValueIterator<String, Long> iterator, KeyPrefixFilter filter) {
330322
this.containerID = containerID;
331323
this.blockLocalIdIterator = iterator;
332324
this.localIdFilter = filter;

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.hadoop.ozone.container.ozoneimpl;
1919

20+
import static org.apache.hadoop.hdds.utils.db.Table.KeyValueIterator.Type.KEY_ONLY;
2021
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
2122
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
2223
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_WORKERS;
@@ -65,7 +66,6 @@
6566
import org.apache.hadoop.hdds.utils.HddsServerUtil;
6667
import org.apache.hadoop.hdds.utils.IOUtils;
6768
import org.apache.hadoop.hdds.utils.db.Table;
68-
import org.apache.hadoop.hdds.utils.db.TableIterator;
6969
import org.apache.hadoop.ozone.HddsDatanodeService;
7070
import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage;
7171
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
@@ -346,8 +346,7 @@ public void buildContainerSet() throws IOException {
346346
for (Thread volumeThread : volumeThreads) {
347347
volumeThread.join();
348348
}
349-
try (TableIterator<ContainerID, ? extends Table.KeyValue<ContainerID, String>> itr =
350-
containerSet.getContainerIdsTable().iterator()) {
349+
try (Table.KeyValueIterator<ContainerID, String> itr = containerSet.getContainerIdsTable().iterator(KEY_ONLY)) {
351350
final Map<ContainerID, Long> containerIds = new HashMap<>();
352351
while (itr.hasNext()) {
353352
containerIds.put(itr.next().getKey(), 0L);

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,13 @@
5959
import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
6060
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
6161
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
62+
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
6263
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
6364
import org.apache.hadoop.hdds.utils.BackgroundService;
6465
import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
6566
import org.apache.hadoop.hdds.utils.db.BatchOperation;
6667
import org.apache.hadoop.hdds.utils.db.CodecBuffer;
6768
import org.apache.hadoop.hdds.utils.db.Table;
68-
import org.apache.hadoop.hdds.utils.db.TableIterator;
6969
import org.apache.hadoop.ozone.common.Checksum;
7070
import org.apache.hadoop.ozone.common.ChunkBuffer;
7171
import org.apache.hadoop.ozone.container.ContainerTestHelper;
@@ -371,9 +371,7 @@ private int getUnderDeletionBlocksCount(DBHandle meta,
371371
DatanodeStore ds = meta.getStore();
372372
DatanodeStoreSchemaTwoImpl dnStoreTwoImpl =
373373
(DatanodeStoreSchemaTwoImpl) ds;
374-
try (
375-
TableIterator<Long, ? extends Table.KeyValue<Long,
376-
StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction>>
374+
try (Table.KeyValueIterator<Long, DeletedBlocksTransaction>
377375
iter = dnStoreTwoImpl.getDeleteTransactionTable().iterator()) {
378376
while (iter.hasNext()) {
379377
StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction
@@ -387,9 +385,7 @@ private int getUnderDeletionBlocksCount(DBHandle meta,
387385
DatanodeStore ds = meta.getStore();
388386
DatanodeStoreSchemaThreeImpl dnStoreThreeImpl =
389387
(DatanodeStoreSchemaThreeImpl) ds;
390-
try (
391-
TableIterator<String, ? extends Table.KeyValue<String,
392-
StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction>>
388+
try (Table.KeyValueIterator<String, DeletedBlocksTransaction>
393389
iter = dnStoreThreeImpl.getDeleteTransactionTable()
394390
.iterator(data.containerPrefix())) {
395391
while (iter.hasNext()) {

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ public void putWithBatch(BatchOperation batch, byte[] key, byte[] value) {
9191

9292
@Override
9393
public boolean isEmpty() throws RocksDatabaseException {
94-
try (KeyValueIterator<byte[], byte[]> keyIter = iterator((byte[]) null, KeyValueIterator.Type.NEITHER)) {
94+
try (KeyValueIterator<byte[], byte[]> keyIter = iterator((byte[])null, KeyValueIterator.Type.NEITHER)) {
9595
keyIter.seekToFirst();
9696
return !keyIter.hasNext();
9797
}

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,16 +142,22 @@ default VALUE getReadCopy(KEY key) throws RocksDatabaseException, CodecException
142142
*/
143143
void deleteRange(KEY beginKey, KEY endKey) throws RocksDatabaseException, CodecException;
144144

145-
/** The same as iterator(null). */
145+
/** The same as iterator(null, KEY_AND_VALUE). */
146146
default KeyValueIterator<KEY, VALUE> iterator() throws RocksDatabaseException, CodecException {
147-
return iterator(null);
147+
return iterator(null, KeyValueIterator.Type.KEY_AND_VALUE);
148148
}
149149

150150
/** The same as iterator(prefix, KEY_AND_VALUE). */
151151
default KeyValueIterator<KEY, VALUE> iterator(KEY prefix) throws RocksDatabaseException, CodecException {
152152
return iterator(prefix, KeyValueIterator.Type.KEY_AND_VALUE);
153153
}
154154

155+
/** The same as iterator(null, type). */
156+
default KeyValueIterator<KEY, VALUE> iterator(KeyValueIterator.Type type)
157+
throws RocksDatabaseException, CodecException {
158+
return iterator(null, type);
159+
}
160+
155161
/**
156162
* Iterate the elements in this table.
157163
*

‎hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestTypedTable.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -156,10 +156,10 @@ public void testContainerIDvsLong() throws Exception {
156156
// test iterator type
157157
try (TypedTable<Long, String> longTable = newTypedTable(
158158
1, LongCodec.get(), StringCodec.get());
159-
Table.KeyValueIterator<Long, String> neither = longTable.iterator(null, NEITHER);
160-
Table.KeyValueIterator<Long, String> keyOnly = longTable.iterator(null, KEY_ONLY);
161-
Table.KeyValueIterator<Long, String> valueOnly = longTable.iterator(null, VALUE_ONLY);
162-
Table.KeyValueIterator<Long, String> keyAndValue = longTable.iterator(null, KEY_AND_VALUE)) {
159+
Table.KeyValueIterator<Long, String> neither = longTable.iterator(NEITHER);
160+
Table.KeyValueIterator<Long, String> keyOnly = longTable.iterator(KEY_ONLY);
161+
Table.KeyValueIterator<Long, String> valueOnly = longTable.iterator(VALUE_ONLY);
162+
Table.KeyValueIterator<Long, String> keyAndValue = longTable.iterator(KEY_AND_VALUE)) {
163163
while (keyAndValue.hasNext()) {
164164
final Table.KeyValue<Long, String> keyValue = keyAndValue.next();
165165
final Long expectedKey = Objects.requireNonNull(keyValue.getKey());

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@
5757
import org.apache.hadoop.hdds.server.events.EventHandler;
5858
import org.apache.hadoop.hdds.server.events.EventPublisher;
5959
import org.apache.hadoop.hdds.utils.db.Table;
60-
import org.apache.hadoop.hdds.utils.db.TableIterator;
6160
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
6261
import org.slf4j.Logger;
6362
import org.slf4j.LoggerFactory;
@@ -137,8 +136,7 @@ public List<DeletedBlocksTransaction> getFailedTransactions(int count,
137136
lock.lock();
138137
try {
139138
final List<DeletedBlocksTransaction> failedTXs = Lists.newArrayList();
140-
try (TableIterator<Long,
141-
? extends Table.KeyValue<Long, DeletedBlocksTransaction>> iter =
139+
try (Table.KeyValueIterator<Long, DeletedBlocksTransaction> iter =
142140
deletedBlockLogStateManager.getReadOnlyIterator()) {
143141
if (count == LIST_ALL_FAILED_TRANSACTIONS) {
144142
while (iter.hasNext()) {
@@ -245,8 +243,7 @@ public int getNumOfValidTransactions() throws IOException {
245243
lock.lock();
246244
try {
247245
final AtomicInteger num = new AtomicInteger(0);
248-
try (TableIterator<Long,
249-
? extends Table.KeyValue<Long, DeletedBlocksTransaction>> iter =
246+
try (Table.KeyValueIterator<Long, DeletedBlocksTransaction> iter =
250247
deletedBlockLogStateManager.getReadOnlyIterator()) {
251248
while (iter.hasNext()) {
252249
DeletedBlocksTransaction delTX = iter.next().getValue();
@@ -394,8 +391,7 @@ public DatanodeDeletedBlockTransactions getTransactions(
394391
scmCommandTimeoutMs);
395392
DatanodeDeletedBlockTransactions transactions =
396393
new DatanodeDeletedBlockTransactions();
397-
try (TableIterator<Long,
398-
? extends Table.KeyValue<Long, DeletedBlocksTransaction>> iter =
394+
try (Table.KeyValueIterator<Long, DeletedBlocksTransaction> iter =
399395
deletedBlockLogStateManager.getReadOnlyIterator()) {
400396
if (lastProcessedTransactionId != -1) {
401397
iter.seek(lastProcessedTransactionId);

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManager.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@
2222
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
2323
import org.apache.hadoop.hdds.scm.metadata.Replicate;
2424
import org.apache.hadoop.hdds.utils.db.Table;
25-
import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
26-
import org.apache.hadoop.hdds.utils.db.TableIterator;
2725

2826
/**
2927
* DeletedBlockLogStateManager interface to
@@ -46,8 +44,7 @@ void increaseRetryCountOfTransactionInDB(ArrayList<Long> txIDs)
4644
int resetRetryCountOfTransactionInDB(ArrayList<Long> txIDs)
4745
throws IOException;
4846

49-
TableIterator<Long,
50-
KeyValue<Long, DeletedBlocksTransaction>> getReadOnlyIterator()
47+
Table.KeyValueIterator<Long, DeletedBlocksTransaction> getReadOnlyIterator()
5148
throws IOException;
5249

5350
void onFlush();

0 commit comments

Comments
 (0)