From e25e461fa8297a5f46fbb7fa0b22fdef0004f398 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Wed, 3 Dec 2025 22:46:45 +0530 Subject: [PATCH 01/13] HDDS-13311. Directory Deleting Service can use deleteRange to delete subDirectories and subFiles. --- .../src/main/proto/OmClientProtocol.proto | 2 + .../hadoop/ozone/om/DeleteKeysResult.java | 31 ++- .../hadoop/ozone/om/KeyManagerImpl.java | 23 +- .../OMDirectoriesPurgeResponseWithFSO.java | 16 +- .../om/service/DirectoryDeletingService.java | 24 +- ...tOMDirectoriesPurgeRequestAndResponse.java | 212 +++++++++++++++++- 6 files changed, 294 insertions(+), 14 deletions(-) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index bdb3cc3cee35..e970faf9ea93 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1473,6 +1473,8 @@ message PurgePathRequest { optional string deletedDir = 3; repeated KeyInfo deletedSubFiles = 4; repeated KeyInfo markDeletedSubDirs = 5; + repeated hadoop.hdds.KeyValue deleteRangeSubFiles = 6; + repeated hadoop.hdds.KeyValue deleteRangeSubDirs = 7; } message DeleteOpenKeysRequest { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java index 2b685edf273d..0f0731e0a6bd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java @@ -27,12 +27,13 @@ public class DeleteKeysResult { private List keysToDelete; - private boolean processedKeys; + private List keyRanges; - public DeleteKeysResult(List keysToDelete, boolean processedKeys) { + public DeleteKeysResult(List keysToDelete, List keyRanges, boolean processedKeys) { this.keysToDelete = keysToDelete; this.processedKeys = processedKeys; + this.keyRanges = keyRanges; } public List getKeysToDelete() { @@ -43,4 +44,30 @@ public boolean isProcessedKeys() { return processedKeys; } + public List getKeyRanges() { + return keyRanges; + } + + /** + * Represents a half-open key range {@code [startKey, exclusiveEndKey)} used + * for RocksDB deleteRange operations. + */ + public static class ExclusiveRange { + private final String startKey; + private final String exclusiveEndKey; + + public ExclusiveRange(String startKey, String exclusiveEndKey) { + this.startKey = startKey; + this.exclusiveEndKey = exclusiveEndKey; + } + + public String getExclusiveEndKey() { + return exclusiveEndKey; + } + + public String getStartKey() { + return startKey; + } + } + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index ece197a2b404..ffea2d2a68a5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT; +import static org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto.READ; import static org.apache.hadoop.hdds.scm.net.NetConstants.NODE_COST_DEFAULT; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getRemoteUser; @@ -2295,15 +2296,33 @@ private DeleteKeysResult gatherSubPathsWithIterat List keyInfos = new ArrayList<>(); String seekFileInDB = metadataManager.getOzonePathKey(volumeId, bucketId, parentInfo.getObjectID(), ""); try (TableIterator> iterator = table.iterator(seekFileInDB)) { - while (iterator.hasNext() && remainingNum > 0) { + String startKey = null; + String lastLoopExclusiveKey = getLexicographicallyHigherString(seekFileInDB); + List keyRanges = new ArrayList<>(); + while (iterator.hasNext()) { KeyValue entry = iterator.next(); KeyValue keyInfo = deleteKeyTransformer.apply(entry); + if (remainingNum <= 0) { + lastLoopExclusiveKey = keyInfo.getKey(); + break; + } if (deleteKeyFilter.apply(keyInfo)) { keyInfos.add(keyInfo.getValue()); remainingNum--; + if (startKey == null) { + startKey = keyInfo.getKey(); + } + } else { + if (startKey != null) { + keyRanges.add(new DeleteKeysResult.ExclusiveRange(startKey, keyInfo.getKey())); + } + startKey = null; } } - return new DeleteKeysResult(keyInfos, !iterator.hasNext()); + if (startKey != null) { + keyRanges.add(new DeleteKeysResult.ExclusiveRange(startKey, lastLoopExclusiveKey)); + } + return new DeleteKeysResult(keyInfos, keyRanges, !iterator.hasNext()); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java index 127ec4ed2cbf..07c15122cea3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.UUID; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.ozone.OmUtils; @@ -147,22 +148,22 @@ public void processPaths( deletedSpaceOmMetadataManager.getDeletedDirTable().putWithBatch(deletedSpaceBatchOperation, ozoneDeleteKey, keyInfo); - keySpaceOmMetadataManager.getDirectoryTable().deleteWithBatch(keySpaceBatchOperation, - ozoneDbKey); - if (LOG.isDebugEnabled()) { LOG.debug("markDeletedDirList KeyName: {}, DBKey: {}", keyInfo.getKeyName(), ozoneDbKey); } } + for (HddsProtos.KeyValue keyRanges : path.getDeleteRangeSubDirsList()) { + keySpaceOmMetadataManager.getDirectoryTable() + .deleteRangeWithBatch(keySpaceBatchOperation, keyRanges.getKey(), keyRanges.getValue()); + } + for (OzoneManagerProtocolProtos.KeyInfo key : deletedSubFilesList) { OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key) .withCommittedKeyDeletedFlag(true); String ozoneDbKey = keySpaceOmMetadataManager.getOzonePathKey(volumeId, bucketId, keyInfo.getParentObjectID(), keyInfo.getFileName()); - keySpaceOmMetadataManager.getKeyTable(getBucketLayout()) - .deleteWithBatch(keySpaceBatchOperation, ozoneDbKey); if (LOG.isDebugEnabled()) { LOG.info("Move keyName:{} to DeletedTable DBKey: {}", @@ -182,6 +183,11 @@ public void processPaths( deletedKey, repeatedOmKeyInfo); } + for (HddsProtos.KeyValue keyRanges : path.getDeleteRangeSubFilesList()) { + keySpaceOmMetadataManager.getKeyTable(getBucketLayout()) + .deleteRangeWithBatch(keySpaceBatchOperation, keyRanges.getKey(), keyRanges.getValue()); + } + if (!openKeyInfoMap.isEmpty()) { for (Map.Entry entry : openKeyInfoMap.entrySet()) { keySpaceOmMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java index a79eeda74f26..71d0424121cc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.ReconfigurationHandler; import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskResult; import org.apache.hadoop.hdds.utils.IOUtils; @@ -436,8 +437,9 @@ private Optional prepareDeleteDirRequest( if (purgeDeletedDir != null) { remainNum.addAndGet(-1); } - return Optional.of(wrapPurgeRequest(volumeBucketId.getVolumeId(), volumeBucketId.getBucketId(), - purgeDeletedDir, subFiles, subDirs)); + return Optional.of( + wrapPurgeRequest(volumeBucketId.getVolumeId(), volumeBucketId.getBucketId(), purgeDeletedDir, subFiles, subDirs, + subDirDeleteResult.getKeyRanges(), subFileDeleteResult.getKeyRanges())); } private OzoneManagerProtocolProtos.PurgePathRequest wrapPurgeRequest( @@ -445,7 +447,9 @@ private OzoneManagerProtocolProtos.PurgePathRequest wrapPurgeRequest( final long bucketId, final String purgeDeletedDir, final List purgeDeletedFiles, - final List markDirsAsDeleted) { + final List markDirsAsDeleted, + List dirExclusiveRanges, + List fileExclusiveRanges) { // Put all keys to be purged in a list PurgePathRequest.Builder purgePathsRequest = PurgePathRequest.newBuilder(); purgePathsRequest.setVolumeId(volumeId); @@ -467,6 +471,20 @@ private OzoneManagerProtocolProtos.PurgePathRequest wrapPurgeRequest( dir.getProtobuf(ClientVersion.CURRENT_VERSION)); } + if (dirExclusiveRanges != null) { + for (DeleteKeysResult.ExclusiveRange range : dirExclusiveRanges) { + purgePathsRequest.addDeleteRangeSubDirs( + HddsProtos.KeyValue.newBuilder().setKey(range.getStartKey()).setValue(range.getExclusiveEndKey()).build()); + } + } + + if (fileExclusiveRanges != null) { + for (DeleteKeysResult.ExclusiveRange range : fileExclusiveRanges) { + purgePathsRequest.addDeleteRangeSubFiles( + HddsProtos.KeyValue.newBuilder().setKey(range.getStartKey()).setValue(range.getExclusiveEndKey()).build()); + } + } + return purgePathsRequest.build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java index a168b7bf380f..4458977da0b0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java @@ -195,9 +195,19 @@ private OMRequest createPurgeKeysRequest(String fromSnapshot, String purgeDelete private PurgePathRequest wrapPurgeRequest( final long volumeId, final long bucketId, final String purgeDeletedDir, final List purgeDeletedFiles, final List markDirsAsDeleted) { + return wrapPurgeRequest( + volumeId, bucketId, purgeDeletedDir, purgeDeletedFiles, markDirsAsDeleted, + null, null); + } + + private PurgePathRequest wrapPurgeRequest( + final long volumeId, final long bucketId, final String purgeDeletedDir, + final List purgeDeletedFiles, final List markDirsAsDeleted, + final List deleteRangeSubDirs, + final List deleteRangeSubFiles) { + // Put all keys to be purged in a list - PurgePathRequest.Builder purgePathsRequest - = PurgePathRequest.newBuilder(); + PurgePathRequest.Builder purgePathsRequest = PurgePathRequest.newBuilder(); purgePathsRequest.setVolumeId(volumeId); purgePathsRequest.setBucketId(bucketId); @@ -217,6 +227,13 @@ private PurgePathRequest wrapPurgeRequest( dir.getProtobuf(ClientVersion.CURRENT_VERSION)); } + if (deleteRangeSubDirs != null) { + purgePathsRequest.addAllDeleteRangeSubDirs(deleteRangeSubDirs); + } + if (deleteRangeSubFiles != null) { + purgePathsRequest.addAllDeleteRangeSubFiles(deleteRangeSubFiles); + } + return purgePathsRequest.build(); } @@ -618,6 +635,197 @@ public void testValidateAndUpdateCacheQuotaBucketRecreated() validateDeletedKeys(omMetadataManager, deletedKeyNames); } + @Test + public void testDeleteRangeSubFilesRespectedByPurge() throws Exception { + when(ozoneManager.getDefaultReplicationConfig()) + .thenReturn(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)); + + String bucket = "bucket" + RandomUtils.secure().randomInt(); + OMRequestTestUtils.addVolumeAndBucketToDB( + volumeName, bucket, omMetadataManager, BucketLayout.FILE_SYSTEM_OPTIMIZED); + + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucket); + OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(bucketKey); + + // Create parent directory "dir1" + OmDirectoryInfo dir1 = new OmDirectoryInfo.Builder() + .setName("dir1") + .setCreationTime(Time.now()) + .setModificationTime(Time.now()) + .setObjectID(1) + .setParentObjectID(bucketInfo.getObjectID()) + .setUpdateID(0) + .build(); + String dirKey = OMRequestTestUtils.addDirKeyToDirTable( + false, dir1, volumeName, bucket, 1L, omMetadataManager); + + // Create 5 files under dir1 + List fileDbKeys = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + OmKeyInfo subFile = OMRequestTestUtils + .createOmKeyInfo(volumeName, bucket, "file" + i, + RatisReplicationConfig.getInstance(ONE)) + .setObjectID(10 + i) + .setParentObjectID(dir1.getObjectID()) + .setUpdateID(100L) + .build(); + + String dbKey = OMRequestTestUtils.addFileToKeyTable( + false, true, subFile.getKeyName(), subFile, 1234L, 10 + i, omMetadataManager); + fileDbKeys.add(dbKey); + } + + // Check: all 5 file keys exist before purge + for (String k : fileDbKeys) { + assertTrue(omMetadataManager.getFileTable().isExist(k)); + } + + // Build deleteRangeSubFiles: + // [file1, file3) and [file4, highKey) so that file1, file2, file4, file5 are purged, + // while file3 (the "non-reclaimable" middle entry) stays. + List fileRanges = new ArrayList<>(); + fileRanges.add(HddsProtos.KeyValue.newBuilder() + .setKey(fileDbKeys.get(0)) + .setValue(fileDbKeys.get(2)) + .build()); + + String highKey = org.apache.hadoop.hdds.StringUtils + .getLexicographicallyHigherString(fileDbKeys.get(4)); + fileRanges.add(HddsProtos.KeyValue.newBuilder() + .setKey(fileDbKeys.get(3)) + .setValue(highKey) + .build()); + + Long volumeId = omMetadataManager.getVolumeId(bucketInfo.getVolumeName()); + Long bucketId = bucketInfo.getObjectID(); + + PurgePathRequest purgePathRequest = wrapPurgeRequest( + volumeId, bucketId, dirKey, + Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), fileRanges); + + List purgePathRequests = Collections.singletonList(purgePathRequest); + List bucketInfoList = Collections.singletonList( + BucketNameInfo.newBuilder() + .setVolumeName(bucketInfo.getVolumeName()) + .setBucketName(bucketInfo.getBucketName()) + .setBucketId(bucketId) + .setVolumeId(volumeId) + .build()); + + OMRequest omRequest = createPurgeKeysRequest( + null, purgePathRequests, bucketInfoList); + OMRequest preExecutedRequest = preExecute(omRequest); + + OzoneManagerProtocolProtos.PurgeDirectoriesRequest dirReq = + preExecutedRequest.getPurgeDirectoriesRequest(); + assertEquals(1, dirReq.getDeletedPathCount()); + assertEquals(2, dirReq.getDeletedPath(0).getDeleteRangeSubFilesCount()); + + OzoneManagerProtocolProtos.PurgePathRequest path = dirReq.getDeletedPath(0); + + // We expect two ranges: [file1, file3) and [file4, highKey) + assertEquals(2, path.getDeleteRangeSubFilesCount()); + assertEquals(fileDbKeys.get(0), path.getDeleteRangeSubFiles(0).getKey()); + assertEquals(fileDbKeys.get(2), path.getDeleteRangeSubFiles(0).getValue()); + assertEquals(fileDbKeys.get(3), path.getDeleteRangeSubFiles(1).getKey()); + assertEquals(highKey, path.getDeleteRangeSubFiles(1).getValue()); + } + + @Test + public void testDeleteRangeSubDirsRespectedByPurge() throws Exception { + when(ozoneManager.getDefaultReplicationConfig()) + .thenReturn(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)); + + String bucket = "bucket" + RandomUtils.secure().randomInt(); + OMRequestTestUtils.addVolumeAndBucketToDB( + volumeName, bucket, omMetadataManager, BucketLayout.FILE_SYSTEM_OPTIMIZED); + + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucket); + OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(bucketKey); + + // Parent directory "dir1" + OmDirectoryInfo dir1 = new OmDirectoryInfo.Builder() + .setName("dir1") + .setCreationTime(Time.now()) + .setModificationTime(Time.now()) + .setObjectID(1) + .setParentObjectID(bucketInfo.getObjectID()) + .setUpdateID(0) + .build(); + String dirKey = OMRequestTestUtils.addDirKeyToDirTable( + false, dir1, volumeName, bucket, 1L, omMetadataManager); + + // Create 5 subdirectories under dir1 + List subDirDbKeys = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + OmDirectoryInfo subdir = new OmDirectoryInfo.Builder() + .setName("subdir" + i) + .setCreationTime(Time.now()) + .setModificationTime(Time.now()) + .setObjectID(10 + i) + .setParentObjectID(dir1.getObjectID()) + .setUpdateID(0) + .build(); + String subDirPath = OMRequestTestUtils.addDirKeyToDirTable( + false, subdir, volumeName, bucket, 2L + i, omMetadataManager); + subDirDbKeys.add(subDirPath); + } + + // Check: all 5 subdir keys exist before purge + for (String k : subDirDbKeys) { + assertTrue(omMetadataManager.getDirectoryTable().isExist(k)); + } + + // Build deleteRangeSubDirs ranges like [subdir1, subdir3) and [subdir4, highKey) + List dirRanges = new ArrayList<>(); + dirRanges.add(HddsProtos.KeyValue.newBuilder() + .setKey(subDirDbKeys.get(0)) + .setValue(subDirDbKeys.get(2)) + .build()); + + String highKey = org.apache.hadoop.hdds.StringUtils + .getLexicographicallyHigherString(subDirDbKeys.get(4)); + dirRanges.add(HddsProtos.KeyValue.newBuilder() + .setKey(subDirDbKeys.get(3)) + .setValue(highKey) + .build()); + + Long volumeId = omMetadataManager.getVolumeId(bucketInfo.getVolumeName()); + Long bucketId = bucketInfo.getObjectID(); + + PurgePathRequest purgePathRequest = wrapPurgeRequest( + volumeId, bucketId, dirKey, + Collections.emptyList(), Collections.emptyList(), + dirRanges, Collections.emptyList()); + + List purgePathRequests = + Collections.singletonList(purgePathRequest); + List bucketInfoList = Collections.singletonList( + BucketNameInfo.newBuilder() + .setVolumeName(bucketInfo.getVolumeName()) + .setBucketName(bucketInfo.getBucketName()) + .setBucketId(bucketId) + .setVolumeId(volumeId) + .build()); + + OMRequest omRequest = createPurgeKeysRequest( + null, purgePathRequests, bucketInfoList); + OMRequest preExecutedRequest = preExecute(omRequest); + + OzoneManagerProtocolProtos.PurgeDirectoriesRequest dirReq = + preExecutedRequest.getPurgeDirectoriesRequest(); + assertEquals(1, dirReq.getDeletedPathCount()); + OzoneManagerProtocolProtos.PurgePathRequest path = dirReq.getDeletedPath(0); + + // We expect two directory ranges: [subdir1, subdir3) and [subdir4, highKey) + assertEquals(2, path.getDeleteRangeSubDirsCount()); + assertEquals(subDirDbKeys.get(0), path.getDeleteRangeSubDirs(0).getKey()); + assertEquals(subDirDbKeys.get(2), path.getDeleteRangeSubDirs(0).getValue()); + assertEquals(subDirDbKeys.get(3), path.getDeleteRangeSubDirs(1).getKey()); + assertEquals(highKey, path.getDeleteRangeSubDirs(1).getValue()); + } + private void performBatchOperationCommit(OMDirectoriesPurgeResponseWithFSO omClientResponse) throws ExecutionException, InterruptedException { CompletableFuture future = new CompletableFuture<>(); From 4c804dd5b2d6c7300220a3013e6f09c0da2b2818 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Thu, 4 Dec 2025 00:58:25 +0530 Subject: [PATCH 02/13] Addressed comment. --- .../main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index ffea2d2a68a5..1859286a8fb7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -2299,11 +2299,13 @@ private DeleteKeysResult gatherSubPathsWithIterat String startKey = null; String lastLoopExclusiveKey = getLexicographicallyHigherString(seekFileInDB); List keyRanges = new ArrayList<>(); + boolean processedAllKeys = true; while (iterator.hasNext()) { KeyValue entry = iterator.next(); KeyValue keyInfo = deleteKeyTransformer.apply(entry); if (remainingNum <= 0) { lastLoopExclusiveKey = keyInfo.getKey(); + processedAllKeys = false; break; } if (deleteKeyFilter.apply(keyInfo)) { @@ -2322,7 +2324,7 @@ private DeleteKeysResult gatherSubPathsWithIterat if (startKey != null) { keyRanges.add(new DeleteKeysResult.ExclusiveRange(startKey, lastLoopExclusiveKey)); } - return new DeleteKeysResult(keyInfos, keyRanges, !iterator.hasNext()); + return new DeleteKeysResult(keyInfos, keyRanges, processedAllKeys); } } From 65c5bc91660704cbcbf6900199f5cce8bd016371 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Thu, 4 Dec 2025 01:35:55 +0530 Subject: [PATCH 03/13] Fixed tests. --- .../java/org/apache/hadoop/ozone/om/KeyManagerImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 1859286a8fb7..fc3deea40de2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -2303,8 +2303,9 @@ private DeleteKeysResult gatherSubPathsWithIterat while (iterator.hasNext()) { KeyValue entry = iterator.next(); KeyValue keyInfo = deleteKeyTransformer.apply(entry); + String tableKey = entry.getKey(); if (remainingNum <= 0) { - lastLoopExclusiveKey = keyInfo.getKey(); + lastLoopExclusiveKey = tableKey; processedAllKeys = false; break; } @@ -2312,11 +2313,11 @@ private DeleteKeysResult gatherSubPathsWithIterat keyInfos.add(keyInfo.getValue()); remainingNum--; if (startKey == null) { - startKey = keyInfo.getKey(); + startKey = tableKey; } } else { if (startKey != null) { - keyRanges.add(new DeleteKeysResult.ExclusiveRange(startKey, keyInfo.getKey())); + keyRanges.add(new DeleteKeysResult.ExclusiveRange(startKey, tableKey)); } startKey = null; } From 572bd9a0079ffc753f131219b69526fb504a0556 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Mon, 8 Dec 2025 00:08:35 +0530 Subject: [PATCH 04/13] Added a test and addressed comments. --- .../hadoop/ozone/om/KeyManagerImpl.java | 18 +- .../om/service/DirectoryDeletingService.java | 16 +- .../ozone/om/TestKeyManagerDeleteRanges.java | 155 ++++++++++++++++++ 3 files changed, 167 insertions(+), 22 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerDeleteRanges.java diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index fc3deea40de2..5cd4f92412b0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -2297,33 +2297,27 @@ private DeleteKeysResult gatherSubPathsWithIterat String seekFileInDB = metadataManager.getOzonePathKey(volumeId, bucketId, parentInfo.getObjectID(), ""); try (TableIterator> iterator = table.iterator(seekFileInDB)) { String startKey = null; - String lastLoopExclusiveKey = getLexicographicallyHigherString(seekFileInDB); List keyRanges = new ArrayList<>(); - boolean processedAllKeys = true; - while (iterator.hasNext()) { + while (iterator.hasNext() && remainingNum > 0) { KeyValue entry = iterator.next(); KeyValue keyInfo = deleteKeyTransformer.apply(entry); - String tableKey = entry.getKey(); - if (remainingNum <= 0) { - lastLoopExclusiveKey = tableKey; - processedAllKeys = false; - break; - } if (deleteKeyFilter.apply(keyInfo)) { keyInfos.add(keyInfo.getValue()); remainingNum--; if (startKey == null) { - startKey = tableKey; + startKey = entry.getKey(); } } else { if (startKey != null) { - keyRanges.add(new DeleteKeysResult.ExclusiveRange(startKey, tableKey)); + keyRanges.add(new DeleteKeysResult.ExclusiveRange(startKey, entry.getKey())); } startKey = null; } } + boolean processedAllKeys = !iterator.hasNext(); if (startKey != null) { - keyRanges.add(new DeleteKeysResult.ExclusiveRange(startKey, lastLoopExclusiveKey)); + keyRanges.add(new DeleteKeysResult.ExclusiveRange(startKey, + processedAllKeys ? getLexicographicallyHigherString(seekFileInDB) : iterator.next().getKey())); } return new DeleteKeysResult(keyInfos, keyRanges, processedAllKeys); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java index 71d0424121cc..71c4571fee6e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java @@ -471,18 +471,14 @@ private OzoneManagerProtocolProtos.PurgePathRequest wrapPurgeRequest( dir.getProtobuf(ClientVersion.CURRENT_VERSION)); } - if (dirExclusiveRanges != null) { - for (DeleteKeysResult.ExclusiveRange range : dirExclusiveRanges) { - purgePathsRequest.addDeleteRangeSubDirs( - HddsProtos.KeyValue.newBuilder().setKey(range.getStartKey()).setValue(range.getExclusiveEndKey()).build()); - } + for (DeleteKeysResult.ExclusiveRange range : dirExclusiveRanges) { + purgePathsRequest.addDeleteRangeSubDirs( + HddsProtos.KeyValue.newBuilder().setKey(range.getStartKey()).setValue(range.getExclusiveEndKey()).build()); } - if (fileExclusiveRanges != null) { - for (DeleteKeysResult.ExclusiveRange range : fileExclusiveRanges) { - purgePathsRequest.addDeleteRangeSubFiles( - HddsProtos.KeyValue.newBuilder().setKey(range.getStartKey()).setValue(range.getExclusiveEndKey()).build()); - } + for (DeleteKeysResult.ExclusiveRange range : fileExclusiveRanges) { + purgePathsRequest.addDeleteRangeSubFiles( + HddsProtos.KeyValue.newBuilder().setKey(range.getStartKey()).setValue(range.getExclusiveEndKey()).build()); } return purgePathsRequest.build(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerDeleteRanges.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerDeleteRanges.java new file mode 100644 index 000000000000..0279f235b6cf --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerDeleteRanges.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import static org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString; +import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getOmKeyInfo; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.server.ServerUtils; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.util.Time; +import org.apache.ratis.util.function.CheckedFunction; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Test for RocksDB delete key range API. + */ + +public class TestKeyManagerDeleteRanges { + + @Test + public void testGetPendingDeletionSubFilesBuildsCorrectRanges(@TempDir File metaDir) throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + ServerUtils.setOzoneMetaDirPath(conf, metaDir.getAbsolutePath()); + OmTestManagers managers = new OmTestManagers(conf); + try { + OMMetadataManager metadataManager = managers.getMetadataManager(); + KeyManager keyManager = managers.getKeyManager(); + + String volume = "vol-range-test"; + String bucket = "buck-range-test"; + + // Create volume + FSO bucket directly in OM DB. + OMRequestTestUtils.addVolumeAndBucketToDB( + volume, bucket, metadataManager, BucketLayout.FILE_SYSTEM_OPTIMIZED); + + String bucketKey = metadataManager.getBucketKey(volume, bucket); + OmBucketInfo bucketInfo = metadataManager.getBucketTable().get(bucketKey); + long bucketObjectId = bucketInfo.getObjectID(); + + // Parent directory "dir1" under the bucket. + OmDirectoryInfo dir1 = new OmDirectoryInfo.Builder() + .setName("dir1") + .setCreationTime(Time.now()) + .setModificationTime(Time.now()) + .setObjectID(1L) + .setParentObjectID(bucketObjectId) + .setUpdateID(0L) + .build(); + + // Insert dir1 into the directory table. + String dirDbKey = OMRequestTestUtils.addDirKeyToDirTable( + false, dir1, volume, bucket, 1L, metadataManager); + + OmKeyInfo parentInfo = getOmKeyInfo( + volume, bucket, dir1, "dir1/"); + + // Create 5 files under dir1 in FileTable. + List fileDbKeys = new ArrayList<>(); + List fileInfos = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + OmKeyInfo subFile = OMRequestTestUtils + .createOmKeyInfo(volume, bucket, "file" + i, + managers.getOzoneManager().getDefaultReplicationConfig()) + .setObjectID(10L + i) + .setParentObjectID(dir1.getObjectID()) + .setUpdateID(100L) + .build(); + + String dbKey = OMRequestTestUtils.addFileToKeyTable( + false, true, subFile.getKeyName(), subFile, 1234L, 10L + i, metadataManager); + fileDbKeys.add(dbKey); + fileInfos.add(subFile); + } + + // Check: All 5 file entries exist in the FileTable. + Table fileTable = + metadataManager.getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED); + for (String k : fileDbKeys) { + assertTrue(fileTable.isExist(k)); + } + + long volumeId = metadataManager.getVolumeId(volume); + long bucketId = metadataManager.getBucketId(volume, bucket); + + // Filter pattern: + // file1 (reclaimable) -> true + // file2 (reclaimable) -> true + // file3 (NOT reclaimable) -> false + // file4 (reclaimable) -> true + // file5 (reclaimable) -> true + CheckedFunction, Boolean, java.io.IOException> filter = + kv -> { + String keyName = kv.getValue().getKeyName(); + return !keyName.endsWith("file3"); + }; + + // remainingNum is large: we don't hit the per-iteration limit. + int remainingNum = 10; + + DeleteKeysResult result = keyManager.getPendingDeletionSubFiles( + volumeId, bucketId, parentInfo, filter, remainingNum); + + // 4 reclaimable files: file1, file2, file4, file5 + assertEquals(4, result.getKeysToDelete().size()); + assertTrue(result.isProcessedKeys()); + + // Expect 2 ExclusiveRanges: + // [file1Key, file3Key) and [file4Key, lexHigher(parentPrefix)) + List ranges = result.getKeyRanges(); + assertEquals(2, ranges.size()); + + String parentPrefix = metadataManager.getOzonePathKey( + volumeId, bucketId, parentInfo.getObjectID(), ""); + String expectedHighKey = getLexicographicallyHigherString(parentPrefix); + + DeleteKeysResult.ExclusiveRange r1 = ranges.get(0); + DeleteKeysResult.ExclusiveRange r2 = ranges.get(1); + + assertEquals(fileDbKeys.get(0), r1.getStartKey()); + assertEquals(fileDbKeys.get(2), r1.getExclusiveEndKey()); + + assertEquals(fileDbKeys.get(3), r2.getStartKey()); + assertEquals(expectedHighKey, r2.getExclusiveEndKey()); + } finally { + managers.stop(); + } + } +} From 7164ebb7dce424a92962b06cf37500b6bd0893be Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Mon, 8 Dec 2025 00:35:50 +0530 Subject: [PATCH 05/13] Fixed find bugs. --- .../apache/hadoop/ozone/om/TestKeyManagerDeleteRanges.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerDeleteRanges.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerDeleteRanges.java index 0279f235b6cf..e1a0eed5e77a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerDeleteRanges.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerDeleteRanges.java @@ -74,16 +74,11 @@ public void testGetPendingDeletionSubFilesBuildsCorrectRanges(@TempDir File meta .setUpdateID(0L) .build(); - // Insert dir1 into the directory table. - String dirDbKey = OMRequestTestUtils.addDirKeyToDirTable( - false, dir1, volume, bucket, 1L, metadataManager); - OmKeyInfo parentInfo = getOmKeyInfo( volume, bucket, dir1, "dir1/"); // Create 5 files under dir1 in FileTable. List fileDbKeys = new ArrayList<>(); - List fileInfos = new ArrayList<>(); for (int i = 1; i <= 5; i++) { OmKeyInfo subFile = OMRequestTestUtils .createOmKeyInfo(volume, bucket, "file" + i, @@ -96,7 +91,6 @@ public void testGetPendingDeletionSubFilesBuildsCorrectRanges(@TempDir File meta String dbKey = OMRequestTestUtils.addFileToKeyTable( false, true, subFile.getKeyName(), subFile, 1234L, 10L + i, metadataManager); fileDbKeys.add(dbKey); - fileInfos.add(subFile); } // Check: All 5 file entries exist in the FileTable. From 48f32a5e8291614977fa8760ecc416071601a3ec Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Wed, 10 Dec 2025 16:23:27 +0530 Subject: [PATCH 06/13] Addressed comments and added tests. --- .../hadoop/ozone/om/TestKeyManagerImpl.java | 206 ++++++++++++++++++ .../hadoop/ozone/om/DeleteKeysResult.java | 25 +++ .../OMDirectoriesPurgeResponseWithFSO.java | 10 +- .../ozone/om/TestKeyManagerDeleteRanges.java | 149 ------------- .../service/TestDirectoryDeletingService.java | 106 ++++++++- 5 files changed, 344 insertions(+), 152 deletions(-) delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerDeleteRanges.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index 584752ab8521..aec74bbcaf66 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -35,6 +35,7 @@ import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -107,6 +108,7 @@ import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.hdds.scm.server.SCMConfigurator; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdds.utils.MapBackedTableIterator; import org.apache.hadoop.hdds.utils.db.InMemoryTestTable; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -138,6 +140,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.apache.ratis.util.ExitUtils; +import org.apache.ratis.util.function.CheckedFunction; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -149,6 +152,7 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mockito; /** * Test class for @{@link KeyManagerImpl}. @@ -1723,6 +1727,208 @@ public void testPreviousSnapshotOzoneDirInfo() throws IOException { assertNull(km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKeyDir4).apply(prevKM)); } + @Test + public void testGetPendingDeletionSubFilesAllReclaimableNoLimit() throws Exception { + OzoneConfiguration configuration = new OzoneConfiguration(); + OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); + KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); + + String prefix = "/vol1/buck1/dir1/"; + java.util.NavigableMap values = new java.util.TreeMap<>(); + // Three reclaimable children under the same parent + OmKeyInfo f1 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file1", null).build(); + OmKeyInfo f2 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file2", null).build(); + OmKeyInfo f3 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file3", null).build(); + values.put(prefix + "file1", f1); + values.put(prefix + "file2", f2); + values.put(prefix + "file3", f3); + + @SuppressWarnings("unchecked") Table fileTable = Mockito.mock(Table.class); + mockTableWithEntries(omMetadataManager, fileTable, "getFileTable", prefix, values); + + OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); + + CheckedFunction, Boolean, IOException> filter = kv -> true; + + DeleteKeysResult result = km.getPendingDeletionSubFiles(1L, 1L, parent, filter, 10); + + // All 3 files reclaimable + assertEquals(3, result.getKeysToDelete().size()); + assertTrue(result.isProcessedKeys()); + + List ranges = result.getKeyRanges(); + assertEquals(1, ranges.size()); + assertEquals(prefix + "file1", ranges.get(0).getStartKey()); + // End key must be lexicographically higher than the parent prefix + assertEquals(org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString(prefix), + ranges.get(0).getExclusiveEndKey()); + } + + @Test + public void testGetPendingDeletionSubFilesMixedReclaimableWithGap() throws Exception { + OzoneConfiguration configuration = new OzoneConfiguration(); + OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); + KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); + + String prefix = "/vol1/buck1/dir1/"; + java.util.NavigableMap values = new java.util.TreeMap<>(); + OmKeyInfo f1 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file1", null).build(); + OmKeyInfo f2 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file2", null).build(); + OmKeyInfo f3 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file3", null).build(); + OmKeyInfo f4 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file4", null).build(); + OmKeyInfo f5 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file5", null).build(); + values.put(prefix + "file1", f1); + values.put(prefix + "file2", f2); + values.put(prefix + "file3", f3); + values.put(prefix + "file4", f4); + values.put(prefix + "file5", f5); + + @SuppressWarnings("unchecked") Table fileTable = Mockito.mock(Table.class); + mockTableWithEntries(omMetadataManager, fileTable, "getFileTable", prefix, values); + + OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); + + // file3 is NOT reclaimable; others are + CheckedFunction, Boolean, IOException> filter = + kv -> !kv.getValue().getKeyName().endsWith("file3"); + + DeleteKeysResult result = km.getPendingDeletionSubFiles(1L, 1L, parent, filter, 10); + + assertEquals(4, result.getKeysToDelete().size()); // 1,2,4,5 + assertTrue(result.isProcessedKeys()); + + List ranges = result.getKeyRanges(); + assertEquals(2, ranges.size()); + + DeleteKeysResult.ExclusiveRange r1 = ranges.get(0); + DeleteKeysResult.ExclusiveRange r2 = ranges.get(1); + + assertEquals(prefix + "file1", r1.getStartKey()); + assertEquals(prefix + "file3", r1.getExclusiveEndKey()); + + assertEquals(prefix + "file4", r2.getStartKey()); + assertEquals(org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString(prefix), r2.getExclusiveEndKey()); + } + + @Test + public void testGetPendingDeletionSubFilesLimitHitsInsideRun() throws Exception { + OzoneConfiguration configuration = new OzoneConfiguration(); + OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); + KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); + + String prefix = "/vol1/buck1/dir1/"; + java.util.NavigableMap values = new java.util.TreeMap<>(); + OmKeyInfo f1 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file1", null).build(); + OmKeyInfo f2 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file2", null).build(); + OmKeyInfo f3 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file3", null).build(); + values.put(prefix + "file1", f1); + values.put(prefix + "file2", f2); + values.put(prefix + "file3", f3); + + @SuppressWarnings("unchecked") Table fileTable = Mockito.mock(Table.class); + mockTableWithEntries(omMetadataManager, fileTable, "getFileTable", prefix, values); + + OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); + + CheckedFunction, Boolean, IOException> filter = kv -> true; + + // remainingNum = 2 -> we only pick file1, file2; file3 is still in iterator + DeleteKeysResult result = km.getPendingDeletionSubFiles(1L, 1L, parent, filter, 2); + + assertEquals(2, result.getKeysToDelete().size()); + assertFalse(result.isProcessedKeys()); + + List ranges = result.getKeyRanges(); + assertEquals(1, ranges.size()); + assertEquals(prefix + "file1", ranges.get(0).getStartKey()); + assertEquals(prefix + "file3", ranges.get(0).getExclusiveEndKey()); // [file1, file3) + } + + @Test + public void testGetPendingDeletionSubFilesFirstNonReclaimable() throws Exception { + OzoneConfiguration configuration = new OzoneConfiguration(); + OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); + KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); + + String prefix = "/vol1/buck1/dir1/"; + java.util.NavigableMap values = new java.util.TreeMap<>(); + OmKeyInfo f1 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file1", null).build(); + OmKeyInfo f2 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file2", null).build(); + values.put(prefix + "file1", f1); + values.put(prefix + "file2", f2); + + @SuppressWarnings("unchecked") Table fileTable = Mockito.mock(Table.class); + mockTableWithEntries(omMetadataManager, fileTable, "getFileTable", prefix, values); + + OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); + + // file1 not reclaimable, file2 reclaimable + CheckedFunction, Boolean, IOException> filter = + kv -> kv.getValue().getKeyName().endsWith("file2"); + + DeleteKeysResult result = km.getPendingDeletionSubFiles(1L, 1L, parent, filter, 10); + + assertEquals(1, result.getKeysToDelete().size()); + assertTrue(result.isProcessedKeys()); + + List ranges = result.getKeyRanges(); + assertEquals(1, ranges.size()); + assertEquals(prefix + "file2", ranges.get(0).getStartKey()); + assertEquals(org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString(prefix), + ranges.get(0).getExclusiveEndKey()); + } + + @Test + public void testGetPendingDeletionSubDirsFirstNonReclaimable() throws Exception { + OzoneConfiguration configuration = new OzoneConfiguration(); + OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); + KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); + + String prefix = "/vol1/buck1/dir1/"; + OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); + java.util.NavigableMap values = new java.util.TreeMap<>(); + OmDirectoryInfo d2 = OMRequestTestUtils.createOmDirectoryInfo("dir2", 101, parent.getParentObjectID()); + OmDirectoryInfo d3 = OMRequestTestUtils.createOmDirectoryInfo("dir3", 102, parent.getParentObjectID()); + values.put(prefix + "dir2", d2); + values.put(prefix + "dir3", d3); + + @SuppressWarnings("unchecked") Table dirTable = Mockito.mock(Table.class); + mockTableWithEntries(omMetadataManager, dirTable, "getDirectoryTable", prefix, values); + + CheckedFunction, Boolean, IOException> filter = + kv -> kv.getValue().getKeyName().endsWith("dir3"); + + DeleteKeysResult result = km.getPendingDeletionSubDirs(1L, 1L, parent, filter, 10); + + assertEquals(1, result.getKeysToDelete().size()); + assertTrue(result.isProcessedKeys()); + + List ranges = result.getKeyRanges(); + assertEquals(1, ranges.size()); + assertEquals(prefix + "dir3", ranges.get(0).getStartKey()); + assertEquals(org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString(prefix), + ranges.get(0).getExclusiveEndKey()); + } + + private void mockTableWithEntries(OMMetadataManager omMetadataManager, Table table, String tableGetter, + String seekPrefix, java.util.NavigableMap values) throws Exception { + + // iterator(prefix) should iterate over entries starting with that prefix. + Mockito.when(table.iterator(org.mockito.ArgumentMatchers.anyString())) + .thenAnswer(i -> new MapBackedTableIterator<>(values, i.getArgument(0))); + + if ("getFileTable".equals(tableGetter)) { + Mockito.when(omMetadataManager.getFileTable()).thenReturn((Table) table); + } else if ("getDirectoryTable".equals(tableGetter)) { + Mockito.when(omMetadataManager.getDirectoryTable()).thenReturn((Table) table); + } + + // gatherSubPathsWithIterator calls this to compute the seek prefix. + Mockito.when( + omMetadataManager.getOzonePathKey(anyLong(), anyLong(), anyLong(), org.mockito.ArgumentMatchers.eq(""))) + .thenReturn(seekPrefix); + } + private void initKeyTableForMultipartTest(String keyName, String volume) throws IOException { List locationInfoGroups = new ArrayList<>(); List locationInfoList = new ArrayList<>(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java index 0f0731e0a6bd..48305db486b7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java @@ -17,8 +17,11 @@ package org.apache.hadoop.ozone.om; +import java.util.Collections; import java.util.List; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Used in {@link org.apache.hadoop.ozone.om.service.DirectoryDeletingService} @@ -29,11 +32,33 @@ public class DeleteKeysResult { private List keysToDelete; private boolean processedKeys; private List keyRanges; + private static final Logger LOG = LoggerFactory.getLogger(DeleteKeysResult.class); public DeleteKeysResult(List keysToDelete, List keyRanges, boolean processedKeys) { this.keysToDelete = keysToDelete; this.processedKeys = processedKeys; this.keyRanges = keyRanges; + validateNonOverlappingRanges(); + } + + private void validateNonOverlappingRanges() { + if (keyRanges == null || keyRanges.size() <= 1) { + return; + } + String lastEnd = null; + for (ExclusiveRange range : keyRanges) { + if (range == null || range.getStartKey() == null || range.getExclusiveEndKey() == null) { + continue; + } + if (lastEnd != null && range.getStartKey().compareTo(lastEnd) < 0) { + LOG.warn( + "Overlapping or unsorted delete ranges detected. " + "Clearing keyRanges to avoid incorrect deleteRange. " + + "previousEnd={}, currentStart={}", lastEnd, range.getStartKey()); + keyRanges = Collections.emptyList(); + return; + } + lastEnd = range.getExclusiveEndKey(); + } } public List getKeysToDelete() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java index 07c15122cea3..41812165d547 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java @@ -157,6 +157,10 @@ public void processPaths( for (HddsProtos.KeyValue keyRanges : path.getDeleteRangeSubDirsList()) { keySpaceOmMetadataManager.getDirectoryTable() .deleteRangeWithBatch(keySpaceBatchOperation, keyRanges.getKey(), keyRanges.getValue()); + if (LOG.isDebugEnabled()) { + LOG.debug("Sub Directory delete range Start Key(inclusive): {} and End Key(exclusive): {}", + keyRanges.getKey(), keyRanges.getValue()); + } } for (OzoneManagerProtocolProtos.KeyInfo key : deletedSubFilesList) { @@ -166,7 +170,7 @@ public void processPaths( bucketId, keyInfo.getParentObjectID(), keyInfo.getFileName()); if (LOG.isDebugEnabled()) { - LOG.info("Move keyName:{} to DeletedTable DBKey: {}", + LOG.debug("Move keyName:{} to DeletedTable DBKey: {}", keyInfo.getKeyName(), ozoneDbKey); } @@ -186,6 +190,10 @@ public void processPaths( for (HddsProtos.KeyValue keyRanges : path.getDeleteRangeSubFilesList()) { keySpaceOmMetadataManager.getKeyTable(getBucketLayout()) .deleteRangeWithBatch(keySpaceBatchOperation, keyRanges.getKey(), keyRanges.getValue()); + if (LOG.isDebugEnabled()) { + LOG.debug("Sub File delete range Start Key(inclusive): {} and End Key(exclusive): {}", keyRanges.getKey(), + keyRanges.getValue()); + } } if (!openKeyInfoMap.isEmpty()) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerDeleteRanges.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerDeleteRanges.java deleted file mode 100644 index e1a0eed5e77a..000000000000 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerDeleteRanges.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.om; - -import static org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString; -import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getOmKeyInfo; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.server.ServerUtils; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.hadoop.util.Time; -import org.apache.ratis.util.function.CheckedFunction; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -/** - * Test for RocksDB delete key range API. - */ - -public class TestKeyManagerDeleteRanges { - - @Test - public void testGetPendingDeletionSubFilesBuildsCorrectRanges(@TempDir File metaDir) throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); - ServerUtils.setOzoneMetaDirPath(conf, metaDir.getAbsolutePath()); - OmTestManagers managers = new OmTestManagers(conf); - try { - OMMetadataManager metadataManager = managers.getMetadataManager(); - KeyManager keyManager = managers.getKeyManager(); - - String volume = "vol-range-test"; - String bucket = "buck-range-test"; - - // Create volume + FSO bucket directly in OM DB. - OMRequestTestUtils.addVolumeAndBucketToDB( - volume, bucket, metadataManager, BucketLayout.FILE_SYSTEM_OPTIMIZED); - - String bucketKey = metadataManager.getBucketKey(volume, bucket); - OmBucketInfo bucketInfo = metadataManager.getBucketTable().get(bucketKey); - long bucketObjectId = bucketInfo.getObjectID(); - - // Parent directory "dir1" under the bucket. - OmDirectoryInfo dir1 = new OmDirectoryInfo.Builder() - .setName("dir1") - .setCreationTime(Time.now()) - .setModificationTime(Time.now()) - .setObjectID(1L) - .setParentObjectID(bucketObjectId) - .setUpdateID(0L) - .build(); - - OmKeyInfo parentInfo = getOmKeyInfo( - volume, bucket, dir1, "dir1/"); - - // Create 5 files under dir1 in FileTable. - List fileDbKeys = new ArrayList<>(); - for (int i = 1; i <= 5; i++) { - OmKeyInfo subFile = OMRequestTestUtils - .createOmKeyInfo(volume, bucket, "file" + i, - managers.getOzoneManager().getDefaultReplicationConfig()) - .setObjectID(10L + i) - .setParentObjectID(dir1.getObjectID()) - .setUpdateID(100L) - .build(); - - String dbKey = OMRequestTestUtils.addFileToKeyTable( - false, true, subFile.getKeyName(), subFile, 1234L, 10L + i, metadataManager); - fileDbKeys.add(dbKey); - } - - // Check: All 5 file entries exist in the FileTable. - Table fileTable = - metadataManager.getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED); - for (String k : fileDbKeys) { - assertTrue(fileTable.isExist(k)); - } - - long volumeId = metadataManager.getVolumeId(volume); - long bucketId = metadataManager.getBucketId(volume, bucket); - - // Filter pattern: - // file1 (reclaimable) -> true - // file2 (reclaimable) -> true - // file3 (NOT reclaimable) -> false - // file4 (reclaimable) -> true - // file5 (reclaimable) -> true - CheckedFunction, Boolean, java.io.IOException> filter = - kv -> { - String keyName = kv.getValue().getKeyName(); - return !keyName.endsWith("file3"); - }; - - // remainingNum is large: we don't hit the per-iteration limit. - int remainingNum = 10; - - DeleteKeysResult result = keyManager.getPendingDeletionSubFiles( - volumeId, bucketId, parentInfo, filter, remainingNum); - - // 4 reclaimable files: file1, file2, file4, file5 - assertEquals(4, result.getKeysToDelete().size()); - assertTrue(result.isProcessedKeys()); - - // Expect 2 ExclusiveRanges: - // [file1Key, file3Key) and [file4Key, lexHigher(parentPrefix)) - List ranges = result.getKeyRanges(); - assertEquals(2, ranges.size()); - - String parentPrefix = metadataManager.getOzonePathKey( - volumeId, bucketId, parentInfo.getObjectID(), ""); - String expectedHighKey = getLexicographicallyHigherString(parentPrefix); - - DeleteKeysResult.ExclusiveRange r1 = ranges.get(0); - DeleteKeysResult.ExclusiveRange r2 = ranges.get(1); - - assertEquals(fileDbKeys.get(0), r1.getStartKey()); - assertEquals(fileDbKeys.get(2), r1.getExclusiveEndKey()); - - assertEquals(fileDbKeys.get(3), r2.getStartKey()); - assertEquals(expectedHighKey, r2.getExclusiveEndKey()); - } finally { - managers.stop(); - } - } -} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java index 06b70dca9054..819ca37c07b4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; import org.apache.hadoop.ozone.om.KeyManager; @@ -63,7 +64,6 @@ import org.apache.ratis.util.ExitUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.mockito.MockedStatic; @@ -241,7 +241,6 @@ public void testMultithreadedDirectoryDeletion() throws Exception { } @Test - @DisplayName("DirectoryDeletingService batches PurgeDirectories by Ratis byte limit (via submitRequest spy)") void testPurgeDirectoriesBatching() throws Exception { final int ratisLimitBytes = 2304; @@ -312,4 +311,107 @@ void testPurgeDirectoriesBatching() throws Exception { org.apache.commons.io.FileUtils.deleteDirectory(testDir); } + @Test + void testDeleteRangeFieldsPropagatedToRatis() throws Exception { + final int ratisLimitBytes = 4096; + + OzoneConfiguration conf = new OzoneConfiguration(); + File testDir = Files.createTempDirectory("testDeleteRange").toFile(); + ServerUtils.setOzoneMetaDirPath(conf, testDir.toString()); + conf.setTimeDuration(OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); + conf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT, ratisLimitBytes, StorageUnit.BYTES); + conf.setQuietMode(false); + + OmTestManagers managers = new OmTestManagers(conf); + om = managers.getOzoneManager(); + KeyManager km = managers.getKeyManager(); + + DirectoryDeletingService real = km.getDirDeletingService(); + DirectoryDeletingService dds = Mockito.spy(real); + + List captured = new ArrayList<>(); + Mockito.doAnswer(inv -> { + OzoneManagerProtocolProtos.OMRequest req = inv.getArgument(0); + captured.add(req); + return OzoneManagerProtocolProtos.OMResponse.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.PurgeDirectories) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .build(); + }).when(dds).submitRequest(Mockito.any(OzoneManagerProtocolProtos.OMRequest.class)); + + final long volumeId = 1L, bucketId = 2L; + String prefix = "/vol1/buck1/dir1/"; + + HddsProtos.KeyValue dirRange1 = HddsProtos.KeyValue.newBuilder() + .setKey(prefix + "subdir1") + .setValue(prefix + "subdir3") + .build(); + HddsProtos.KeyValue dirRange2 = HddsProtos.KeyValue.newBuilder() + .setKey(prefix + "subdir4") + .setValue("zzzz") + .build(); + + HddsProtos.KeyValue fileRange1 = HddsProtos.KeyValue.newBuilder() + .setKey(prefix + "file1") + .setValue(prefix + "file3") + .build(); + + OzoneManagerProtocolProtos.PurgePathRequest purgePath = OzoneManagerProtocolProtos.PurgePathRequest.newBuilder() + .setVolumeId(volumeId) + .setBucketId(bucketId) + .setDeletedDir(prefix + "deletedDir") + .addDeleteRangeSubDirs(dirRange1) + .addDeleteRangeSubDirs(dirRange2) + .addDeleteRangeSubFiles(fileRange1) + .build(); + + List purgeList = java.util.Collections.singletonList(purgePath); + + org.apache.hadoop.ozone.om.OMMetadataManager.VolumeBucketId vbId = + new org.apache.hadoop.ozone.om.OMMetadataManager.VolumeBucketId(volumeId, bucketId); + OzoneManagerProtocolProtos.BucketNameInfo bni = + OzoneManagerProtocolProtos.BucketNameInfo.newBuilder() + .setVolumeId(volumeId) + .setBucketId(bucketId) + .setVolumeName("v") + .setBucketName("b") + .build(); + Map + bucketNameInfoMap = new HashMap<>(); + bucketNameInfoMap.put(vbId, bni); + + dds.optimizeDirDeletesAndSubmitRequest( + 0L, 0L, 0L, + new ArrayList<>(), purgeList, + null, Time.monotonicNow(), + km, + kv -> true, kv -> true, + bucketNameInfoMap, + null, 1L, + new AtomicInteger(Integer.MAX_VALUE)); + + // Exactly one PurgeDirectories OMRequest expected + assertThat(captured).hasSize(1); + OzoneManagerProtocolProtos.OMRequest omReq = captured.get(0); + assertThat(omReq.getCmdType()).isEqualTo(OzoneManagerProtocolProtos.Type.PurgeDirectories); + + OzoneManagerProtocolProtos.PurgeDirectoriesRequest purgeReq = omReq.getPurgeDirectoriesRequest(); + assertThat(purgeReq.getDeletedPathCount()).isEqualTo(1); + OzoneManagerProtocolProtos.PurgePathRequest path = purgeReq.getDeletedPath(0); + + // Verify dir ranges + assertThat(path.getDeleteRangeSubDirsCount()).isEqualTo(2); + assertThat(path.getDeleteRangeSubDirs(0).getKey()).isEqualTo(dirRange1.getKey()); + assertThat(path.getDeleteRangeSubDirs(0).getValue()).isEqualTo(dirRange1.getValue()); + assertThat(path.getDeleteRangeSubDirs(1).getKey()).isEqualTo(dirRange2.getKey()); + assertThat(path.getDeleteRangeSubDirs(1).getValue()).isEqualTo(dirRange2.getValue()); + + // Verify file ranges + assertThat(path.getDeleteRangeSubFilesCount()).isEqualTo(1); + assertThat(path.getDeleteRangeSubFiles(0).getKey()).isEqualTo(fileRange1.getKey()); + assertThat(path.getDeleteRangeSubFiles(0).getValue()).isEqualTo(fileRange1.getValue()); + + org.apache.commons.io.FileUtils.deleteDirectory(testDir); + } + } From 4c953231cb05e356ea2b11c4bf4a838b6b83e5f2 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Fri, 12 Dec 2025 13:17:56 +0530 Subject: [PATCH 07/13] Updated tests. --- .../hadoop/ozone/om/TestKeyManagerImpl.java | 206 ------------------ .../hadoop/ozone/om/TestKeyManagerImpl.java | 195 +++++++++++++++++ 2 files changed, 195 insertions(+), 206 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index aec74bbcaf66..584752ab8521 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -35,7 +35,6 @@ import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -108,7 +107,6 @@ import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.hdds.scm.server.SCMConfigurator; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; -import org.apache.hadoop.hdds.utils.MapBackedTableIterator; import org.apache.hadoop.hdds.utils.db.InMemoryTestTable; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -140,7 +138,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.apache.ratis.util.ExitUtils; -import org.apache.ratis.util.function.CheckedFunction; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -152,7 +149,6 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; -import org.mockito.Mockito; /** * Test class for @{@link KeyManagerImpl}. @@ -1727,208 +1723,6 @@ public void testPreviousSnapshotOzoneDirInfo() throws IOException { assertNull(km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKeyDir4).apply(prevKM)); } - @Test - public void testGetPendingDeletionSubFilesAllReclaimableNoLimit() throws Exception { - OzoneConfiguration configuration = new OzoneConfiguration(); - OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); - KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); - - String prefix = "/vol1/buck1/dir1/"; - java.util.NavigableMap values = new java.util.TreeMap<>(); - // Three reclaimable children under the same parent - OmKeyInfo f1 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file1", null).build(); - OmKeyInfo f2 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file2", null).build(); - OmKeyInfo f3 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file3", null).build(); - values.put(prefix + "file1", f1); - values.put(prefix + "file2", f2); - values.put(prefix + "file3", f3); - - @SuppressWarnings("unchecked") Table fileTable = Mockito.mock(Table.class); - mockTableWithEntries(omMetadataManager, fileTable, "getFileTable", prefix, values); - - OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); - - CheckedFunction, Boolean, IOException> filter = kv -> true; - - DeleteKeysResult result = km.getPendingDeletionSubFiles(1L, 1L, parent, filter, 10); - - // All 3 files reclaimable - assertEquals(3, result.getKeysToDelete().size()); - assertTrue(result.isProcessedKeys()); - - List ranges = result.getKeyRanges(); - assertEquals(1, ranges.size()); - assertEquals(prefix + "file1", ranges.get(0).getStartKey()); - // End key must be lexicographically higher than the parent prefix - assertEquals(org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString(prefix), - ranges.get(0).getExclusiveEndKey()); - } - - @Test - public void testGetPendingDeletionSubFilesMixedReclaimableWithGap() throws Exception { - OzoneConfiguration configuration = new OzoneConfiguration(); - OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); - KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); - - String prefix = "/vol1/buck1/dir1/"; - java.util.NavigableMap values = new java.util.TreeMap<>(); - OmKeyInfo f1 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file1", null).build(); - OmKeyInfo f2 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file2", null).build(); - OmKeyInfo f3 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file3", null).build(); - OmKeyInfo f4 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file4", null).build(); - OmKeyInfo f5 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file5", null).build(); - values.put(prefix + "file1", f1); - values.put(prefix + "file2", f2); - values.put(prefix + "file3", f3); - values.put(prefix + "file4", f4); - values.put(prefix + "file5", f5); - - @SuppressWarnings("unchecked") Table fileTable = Mockito.mock(Table.class); - mockTableWithEntries(omMetadataManager, fileTable, "getFileTable", prefix, values); - - OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); - - // file3 is NOT reclaimable; others are - CheckedFunction, Boolean, IOException> filter = - kv -> !kv.getValue().getKeyName().endsWith("file3"); - - DeleteKeysResult result = km.getPendingDeletionSubFiles(1L, 1L, parent, filter, 10); - - assertEquals(4, result.getKeysToDelete().size()); // 1,2,4,5 - assertTrue(result.isProcessedKeys()); - - List ranges = result.getKeyRanges(); - assertEquals(2, ranges.size()); - - DeleteKeysResult.ExclusiveRange r1 = ranges.get(0); - DeleteKeysResult.ExclusiveRange r2 = ranges.get(1); - - assertEquals(prefix + "file1", r1.getStartKey()); - assertEquals(prefix + "file3", r1.getExclusiveEndKey()); - - assertEquals(prefix + "file4", r2.getStartKey()); - assertEquals(org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString(prefix), r2.getExclusiveEndKey()); - } - - @Test - public void testGetPendingDeletionSubFilesLimitHitsInsideRun() throws Exception { - OzoneConfiguration configuration = new OzoneConfiguration(); - OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); - KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); - - String prefix = "/vol1/buck1/dir1/"; - java.util.NavigableMap values = new java.util.TreeMap<>(); - OmKeyInfo f1 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file1", null).build(); - OmKeyInfo f2 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file2", null).build(); - OmKeyInfo f3 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file3", null).build(); - values.put(prefix + "file1", f1); - values.put(prefix + "file2", f2); - values.put(prefix + "file3", f3); - - @SuppressWarnings("unchecked") Table fileTable = Mockito.mock(Table.class); - mockTableWithEntries(omMetadataManager, fileTable, "getFileTable", prefix, values); - - OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); - - CheckedFunction, Boolean, IOException> filter = kv -> true; - - // remainingNum = 2 -> we only pick file1, file2; file3 is still in iterator - DeleteKeysResult result = km.getPendingDeletionSubFiles(1L, 1L, parent, filter, 2); - - assertEquals(2, result.getKeysToDelete().size()); - assertFalse(result.isProcessedKeys()); - - List ranges = result.getKeyRanges(); - assertEquals(1, ranges.size()); - assertEquals(prefix + "file1", ranges.get(0).getStartKey()); - assertEquals(prefix + "file3", ranges.get(0).getExclusiveEndKey()); // [file1, file3) - } - - @Test - public void testGetPendingDeletionSubFilesFirstNonReclaimable() throws Exception { - OzoneConfiguration configuration = new OzoneConfiguration(); - OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); - KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); - - String prefix = "/vol1/buck1/dir1/"; - java.util.NavigableMap values = new java.util.TreeMap<>(); - OmKeyInfo f1 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file1", null).build(); - OmKeyInfo f2 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file2", null).build(); - values.put(prefix + "file1", f1); - values.put(prefix + "file2", f2); - - @SuppressWarnings("unchecked") Table fileTable = Mockito.mock(Table.class); - mockTableWithEntries(omMetadataManager, fileTable, "getFileTable", prefix, values); - - OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); - - // file1 not reclaimable, file2 reclaimable - CheckedFunction, Boolean, IOException> filter = - kv -> kv.getValue().getKeyName().endsWith("file2"); - - DeleteKeysResult result = km.getPendingDeletionSubFiles(1L, 1L, parent, filter, 10); - - assertEquals(1, result.getKeysToDelete().size()); - assertTrue(result.isProcessedKeys()); - - List ranges = result.getKeyRanges(); - assertEquals(1, ranges.size()); - assertEquals(prefix + "file2", ranges.get(0).getStartKey()); - assertEquals(org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString(prefix), - ranges.get(0).getExclusiveEndKey()); - } - - @Test - public void testGetPendingDeletionSubDirsFirstNonReclaimable() throws Exception { - OzoneConfiguration configuration = new OzoneConfiguration(); - OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); - KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); - - String prefix = "/vol1/buck1/dir1/"; - OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); - java.util.NavigableMap values = new java.util.TreeMap<>(); - OmDirectoryInfo d2 = OMRequestTestUtils.createOmDirectoryInfo("dir2", 101, parent.getParentObjectID()); - OmDirectoryInfo d3 = OMRequestTestUtils.createOmDirectoryInfo("dir3", 102, parent.getParentObjectID()); - values.put(prefix + "dir2", d2); - values.put(prefix + "dir3", d3); - - @SuppressWarnings("unchecked") Table dirTable = Mockito.mock(Table.class); - mockTableWithEntries(omMetadataManager, dirTable, "getDirectoryTable", prefix, values); - - CheckedFunction, Boolean, IOException> filter = - kv -> kv.getValue().getKeyName().endsWith("dir3"); - - DeleteKeysResult result = km.getPendingDeletionSubDirs(1L, 1L, parent, filter, 10); - - assertEquals(1, result.getKeysToDelete().size()); - assertTrue(result.isProcessedKeys()); - - List ranges = result.getKeyRanges(); - assertEquals(1, ranges.size()); - assertEquals(prefix + "dir3", ranges.get(0).getStartKey()); - assertEquals(org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString(prefix), - ranges.get(0).getExclusiveEndKey()); - } - - private void mockTableWithEntries(OMMetadataManager omMetadataManager, Table table, String tableGetter, - String seekPrefix, java.util.NavigableMap values) throws Exception { - - // iterator(prefix) should iterate over entries starting with that prefix. - Mockito.when(table.iterator(org.mockito.ArgumentMatchers.anyString())) - .thenAnswer(i -> new MapBackedTableIterator<>(values, i.getArgument(0))); - - if ("getFileTable".equals(tableGetter)) { - Mockito.when(omMetadataManager.getFileTable()).thenReturn((Table) table); - } else if ("getDirectoryTable".equals(tableGetter)) { - Mockito.when(omMetadataManager.getDirectoryTable()).thenReturn((Table) table); - } - - // gatherSubPathsWithIterator calls this to compute the seek prefix. - Mockito.when( - omMetadataManager.getOzonePathKey(anyLong(), anyLong(), anyLong(), org.mockito.ArgumentMatchers.eq(""))) - .thenReturn(seekPrefix); - } - private void initKeyTableForMultipartTest(String keyName, String volume) throws IOException { List locationInfoGroups = new ArrayList<>(); List locationInfoList = new ArrayList<>(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index 52cd9fb15cac..195394c86bef 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -21,7 +21,10 @@ import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.SNAPSHOT_RENAMED_TABLE; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -36,10 +39,14 @@ import java.util.stream.Stream; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.MapBackedTableIterator; +import org.apache.hadoop.hdds.utils.db.StringInMemoryTestTable; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.ratis.util.function.CheckedFunction; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -229,4 +236,192 @@ public void testGetDeletedDirEntries(int numberOfVolumes, int numberOfBucketsPer assertEquals(expectedEntries, km.getDeletedDirEntries(volumeName, bucketName, numberOfEntries)); } } + + @Test + public void testGetPendingDeletionSubFilesAllReclaimableNoLimit() throws Exception { + OzoneConfiguration configuration = new OzoneConfiguration(); + OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); + KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); + + String prefix = "/vol1/buck1/dir1/"; + java.util.NavigableMap values = new java.util.TreeMap<>(); + // Three reclaimable children under the same parent + OmKeyInfo f1 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file1", null).build(); + OmKeyInfo f2 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file2", null).build(); + OmKeyInfo f3 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file3", null).build(); + values.put(prefix + "file1", f1); + values.put(prefix + "file2", f2); + values.put(prefix + "file3", f3); + + Table fileTable = new StringInMemoryTestTable<>(values, "fileTable"); + Mockito.when(omMetadataManager.getFileTable()).thenReturn(fileTable); + Mockito.when(omMetadataManager.getOzonePathKey(anyLong(), anyLong(), anyLong(), eq(""))).thenReturn(prefix); + + OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); + + CheckedFunction, Boolean, IOException> filter = kv -> true; + + DeleteKeysResult result = km.getPendingDeletionSubFiles(1L, 1L, parent, filter, 10); + + // All 3 files reclaimable + assertEquals(3, result.getKeysToDelete().size()); + assertTrue(result.isProcessedKeys()); + + List ranges = result.getKeyRanges(); + assertEquals(1, ranges.size()); + assertEquals(prefix + "file1", ranges.get(0).getStartKey()); + // End key must be lexicographically higher than the parent prefix + assertEquals(org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString(prefix), + ranges.get(0).getExclusiveEndKey()); + } + + @Test + public void testGetPendingDeletionSubFilesMixedReclaimableWithGap() throws Exception { + OzoneConfiguration configuration = new OzoneConfiguration(); + OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); + KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); + + String prefix = "/vol1/buck1/dir1/"; + java.util.NavigableMap values = new java.util.TreeMap<>(); + OmKeyInfo f1 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file1", null).build(); + OmKeyInfo f2 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file2", null).build(); + OmKeyInfo f3 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file3", null).build(); + OmKeyInfo f4 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file4", null).build(); + OmKeyInfo f5 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file5", null).build(); + values.put(prefix + "file1", f1); + values.put(prefix + "file2", f2); + values.put(prefix + "file3", f3); + values.put(prefix + "file4", f4); + values.put(prefix + "file5", f5); + + Table fileTable = new StringInMemoryTestTable<>(values, "fileTable"); + Mockito.when(omMetadataManager.getFileTable()).thenReturn(fileTable); + Mockito.when(omMetadataManager.getOzonePathKey(anyLong(), anyLong(), anyLong(), eq(""))).thenReturn(prefix); + + OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); + + // file3 is NOT reclaimable; others are + CheckedFunction, Boolean, IOException> filter = + kv -> !kv.getValue().getKeyName().endsWith("file3"); + + DeleteKeysResult result = km.getPendingDeletionSubFiles(1L, 1L, parent, filter, 10); + + assertEquals(4, result.getKeysToDelete().size()); // 1,2,4,5 + assertTrue(result.isProcessedKeys()); + + List ranges = result.getKeyRanges(); + assertEquals(2, ranges.size()); + + DeleteKeysResult.ExclusiveRange r1 = ranges.get(0); + DeleteKeysResult.ExclusiveRange r2 = ranges.get(1); + + assertEquals(prefix + "file1", r1.getStartKey()); + assertEquals(prefix + "file3", r1.getExclusiveEndKey()); + + assertEquals(prefix + "file4", r2.getStartKey()); + assertEquals(org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString(prefix), r2.getExclusiveEndKey()); + } + + @Test + public void testGetPendingDeletionSubFilesLimitHitsInsideRun() throws Exception { + OzoneConfiguration configuration = new OzoneConfiguration(); + OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); + KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); + + String prefix = "/vol1/buck1/dir1/"; + java.util.NavigableMap values = new java.util.TreeMap<>(); + OmKeyInfo f1 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file1", null).build(); + OmKeyInfo f2 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file2", null).build(); + OmKeyInfo f3 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file3", null).build(); + values.put(prefix + "file1", f1); + values.put(prefix + "file2", f2); + values.put(prefix + "file3", f3); + + Table fileTable = new StringInMemoryTestTable<>(values, "fileTable"); + Mockito.when(omMetadataManager.getFileTable()).thenReturn(fileTable); + Mockito.when(omMetadataManager.getOzonePathKey(anyLong(), anyLong(), anyLong(), eq(""))).thenReturn(prefix); + + OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); + + CheckedFunction, Boolean, IOException> filter = kv -> true; + + // remainingNum = 2 -> we only pick file1, file2; file3 is still in iterator + DeleteKeysResult result = km.getPendingDeletionSubFiles(1L, 1L, parent, filter, 2); + + assertEquals(2, result.getKeysToDelete().size()); + assertFalse(result.isProcessedKeys()); + + List ranges = result.getKeyRanges(); + assertEquals(1, ranges.size()); + assertEquals(prefix + "file1", ranges.get(0).getStartKey()); + assertEquals(prefix + "file3", ranges.get(0).getExclusiveEndKey()); // [file1, file3) + } + + @Test + public void testGetPendingDeletionSubFilesFirstNonReclaimable() throws Exception { + OzoneConfiguration configuration = new OzoneConfiguration(); + OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); + KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); + + String prefix = "/vol1/buck1/dir1/"; + java.util.NavigableMap values = new java.util.TreeMap<>(); + OmKeyInfo f1 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file1", null).build(); + OmKeyInfo f2 = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "file2", null).build(); + values.put(prefix + "file1", f1); + values.put(prefix + "file2", f2); + + Table fileTable = new StringInMemoryTestTable<>(values, "fileTable"); + Mockito.when(omMetadataManager.getFileTable()).thenReturn(fileTable); + Mockito.when(omMetadataManager.getOzonePathKey(anyLong(), anyLong(), anyLong(), eq(""))).thenReturn(prefix); + + OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); + + // file1 not reclaimable, file2 reclaimable + CheckedFunction, Boolean, IOException> filter = + kv -> kv.getValue().getKeyName().endsWith("file2"); + + DeleteKeysResult result = km.getPendingDeletionSubFiles(1L, 1L, parent, filter, 10); + + assertEquals(1, result.getKeysToDelete().size()); + assertTrue(result.isProcessedKeys()); + + List ranges = result.getKeyRanges(); + assertEquals(1, ranges.size()); + assertEquals(prefix + "file2", ranges.get(0).getStartKey()); + assertEquals(org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString(prefix), + ranges.get(0).getExclusiveEndKey()); + } + + @Test + public void testGetPendingDeletionSubDirsFirstNonReclaimable() throws Exception { + OzoneConfiguration configuration = new OzoneConfiguration(); + OMMetadataManager omMetadataManager = Mockito.mock(OMMetadataManager.class); + KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); + + String prefix = "/vol1/buck1/dir1/"; + OmKeyInfo parent = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", "dir1", null).setObjectID(100L).build(); + java.util.NavigableMap values = new java.util.TreeMap<>(); + OmDirectoryInfo d2 = OMRequestTestUtils.createOmDirectoryInfo("dir2", 101, parent.getParentObjectID()); + OmDirectoryInfo d3 = OMRequestTestUtils.createOmDirectoryInfo("dir3", 102, parent.getParentObjectID()); + values.put(prefix + "dir2", d2); + values.put(prefix + "dir3", d3); + + Table dirTable = new StringInMemoryTestTable<>(values, "directoryTable"); + Mockito.when(omMetadataManager.getDirectoryTable()).thenReturn(dirTable); + Mockito.when(omMetadataManager.getOzonePathKey(anyLong(), anyLong(), anyLong(), eq(""))).thenReturn(prefix); + + CheckedFunction, Boolean, IOException> filter = + kv -> kv.getValue().getKeyName().endsWith("dir3"); + + DeleteKeysResult result = km.getPendingDeletionSubDirs(1L, 1L, parent, filter, 10); + + assertEquals(1, result.getKeysToDelete().size()); + assertTrue(result.isProcessedKeys()); + + List ranges = result.getKeyRanges(); + assertEquals(1, ranges.size()); + assertEquals(prefix + "dir3", ranges.get(0).getStartKey()); + assertEquals(org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString(prefix), + ranges.get(0).getExclusiveEndKey()); + } } From 4c70d2135a865c558da579516dd14e0c06fa855b Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 16 Dec 2025 01:40:07 +0530 Subject: [PATCH 08/13] Addressed comments. --- .../hadoop/ozone/om/DeleteKeysResult.java | 27 +++---------------- .../OMDirectoriesPurgeResponseWithFSO.java | 12 +++------ 2 files changed, 7 insertions(+), 32 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java index 48305db486b7..5312d5ef2822 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java @@ -34,31 +34,10 @@ public class DeleteKeysResult { private List keyRanges; private static final Logger LOG = LoggerFactory.getLogger(DeleteKeysResult.class); - public DeleteKeysResult(List keysToDelete, List keyRanges, boolean processedKeys) { - this.keysToDelete = keysToDelete; + DeleteKeysResult(List keysToDelete, List keyRanges, boolean processedKeys) { + this.keysToDelete = Collections.unmodifiableList(keysToDelete); this.processedKeys = processedKeys; - this.keyRanges = keyRanges; - validateNonOverlappingRanges(); - } - - private void validateNonOverlappingRanges() { - if (keyRanges == null || keyRanges.size() <= 1) { - return; - } - String lastEnd = null; - for (ExclusiveRange range : keyRanges) { - if (range == null || range.getStartKey() == null || range.getExclusiveEndKey() == null) { - continue; - } - if (lastEnd != null && range.getStartKey().compareTo(lastEnd) < 0) { - LOG.warn( - "Overlapping or unsorted delete ranges detected. " + "Clearing keyRanges to avoid incorrect deleteRange. " + - "previousEnd={}, currentStart={}", lastEnd, range.getStartKey()); - keyRanges = Collections.emptyList(); - return; - } - lastEnd = range.getExclusiveEndKey(); - } + this.keyRanges = Collections.unmodifiableList(keyRanges); } public List getKeysToDelete() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java index 41812165d547..5e0fa7de11e1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java @@ -157,10 +157,8 @@ public void processPaths( for (HddsProtos.KeyValue keyRanges : path.getDeleteRangeSubDirsList()) { keySpaceOmMetadataManager.getDirectoryTable() .deleteRangeWithBatch(keySpaceBatchOperation, keyRanges.getKey(), keyRanges.getValue()); - if (LOG.isDebugEnabled()) { - LOG.debug("Sub Directory delete range Start Key(inclusive): {} and End Key(exclusive): {}", - keyRanges.getKey(), keyRanges.getValue()); - } + LOG.debug("Sub Directory delete range Start Key(inclusive): {} and End Key(exclusive): {}", keyRanges.getKey(), + keyRanges.getValue()); } for (OzoneManagerProtocolProtos.KeyInfo key : deletedSubFilesList) { @@ -190,10 +188,8 @@ public void processPaths( for (HddsProtos.KeyValue keyRanges : path.getDeleteRangeSubFilesList()) { keySpaceOmMetadataManager.getKeyTable(getBucketLayout()) .deleteRangeWithBatch(keySpaceBatchOperation, keyRanges.getKey(), keyRanges.getValue()); - if (LOG.isDebugEnabled()) { - LOG.debug("Sub File delete range Start Key(inclusive): {} and End Key(exclusive): {}", keyRanges.getKey(), - keyRanges.getValue()); - } + LOG.debug("Sub File delete range Start Key(inclusive): {} and End Key(exclusive): {}", keyRanges.getKey(), + keyRanges.getValue()); } if (!openKeyInfoMap.isEmpty()) { From e06dd711c0e32a79fc54899b0a93b4d46584fcfd Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 16 Dec 2025 01:46:19 +0530 Subject: [PATCH 09/13] Fix pmd. --- .../main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java index 5312d5ef2822..b9a9d274b4a9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java @@ -32,7 +32,6 @@ public class DeleteKeysResult { private List keysToDelete; private boolean processedKeys; private List keyRanges; - private static final Logger LOG = LoggerFactory.getLogger(DeleteKeysResult.class); DeleteKeysResult(List keysToDelete, List keyRanges, boolean processedKeys) { this.keysToDelete = Collections.unmodifiableList(keysToDelete); From c1c6aac78a0dea38181a616fe03e749722761a16 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 16 Dec 2025 02:37:03 +0530 Subject: [PATCH 10/13] Fixed checkstyle. --- .../main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java index b9a9d274b4a9..39f371f83bdd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java @@ -20,8 +20,6 @@ import java.util.Collections; import java.util.List; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Used in {@link org.apache.hadoop.ozone.om.service.DirectoryDeletingService} From 959365c45f2bce2409347f531e4fc39e1ddd12bd Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Wed, 17 Dec 2025 12:46:49 +0530 Subject: [PATCH 11/13] Added null checks. --- .../org/apache/hadoop/ozone/om/DeleteKeysResult.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java index 39f371f83bdd..9988afbc8fc8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeleteKeysResult.java @@ -32,9 +32,11 @@ public class DeleteKeysResult { private List keyRanges; DeleteKeysResult(List keysToDelete, List keyRanges, boolean processedKeys) { - this.keysToDelete = Collections.unmodifiableList(keysToDelete); + this.keysToDelete = + Collections.unmodifiableList(java.util.Objects.requireNonNull(keysToDelete, "keysToDelete must not be null")); + this.keyRanges = + Collections.unmodifiableList(java.util.Objects.requireNonNull(keyRanges, "keyRanges must not be null")); this.processedKeys = processedKeys; - this.keyRanges = Collections.unmodifiableList(keyRanges); } public List getKeysToDelete() { @@ -58,8 +60,8 @@ public static class ExclusiveRange { private final String exclusiveEndKey; public ExclusiveRange(String startKey, String exclusiveEndKey) { - this.startKey = startKey; - this.exclusiveEndKey = exclusiveEndKey; + this.startKey = java.util.Objects.requireNonNull(startKey, "startKey must not be null"); + this.exclusiveEndKey = java.util.Objects.requireNonNull(exclusiveEndKey, "exclusiveEndKey must not be null"); } public String getExclusiveEndKey() { From eee43c0c5cf705aa09d8c93a3958c549b066b8da Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Fri, 19 Dec 2025 13:57:06 +0530 Subject: [PATCH 12/13] add snapshot diff testcase --- .../ozone/om/snapshot/TestOmSnapshot.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index 585d8a943959..0fed12e14d0f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -27,6 +27,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.COMPACTION_LOG_TABLE; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF; @@ -99,10 +100,13 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CompactionLogEntryProto; import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils; +import org.apache.hadoop.hdds.utils.db.CodecException; import org.apache.hadoop.hdds.utils.db.DBProfile; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.RocksDatabase; +import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; +import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; @@ -137,6 +141,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; @@ -230,6 +235,7 @@ private void init() throws Exception { conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true); conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY, OMLayoutFeature.BUCKET_LAYOUT_SUPPORT.layoutVersion()); conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS); + conf.setTimeDuration(OZONE_DIR_DELETING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS); conf.setInt(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, -1); if (!disableNativeDiff) { conf.setTimeDuration(OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL, 0, TimeUnit.SECONDS); @@ -1076,6 +1082,67 @@ public void testSnapDiffWithDirectoryDelete() throws Exception { assertEquals(diff.getDiffList(), diffEntries); } + /** + * Testing scenario: + * 1) Dir dir1/dir2 is created. + * 2) Snapshot snap1 created. + * 3) Delete dir1. + * 4) Wait for DDS to run and pick the sub-dirs and purge. + * 6) Snapshot snap2 created. + * 5) Snap-diff b/w snapshot snap1 and snap2 should have 1 entry + * in case of native lib (dir1) and 2 entries (dir1, dir1/dir2) + * in case of non-native env. + * This is because native lib impl will read the single entry from + * range delete tombstone. + */ + @Test + public void testSnapDiffWithDirectoryDeleteAfterDDSProcessing() throws Exception { + startKeyManager(); + assumeTrue(bucketLayout.isFileSystemOptimized()); + String testVolumeName = "vol" + counter.incrementAndGet(); + String testBucketName = "bucket1"; + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + createBucket(volume, testBucketName); + OzoneBucket bucket = volume.getBucket(testBucketName); + String snap1 = "snap1"; + String dir1 = "dir1"; + String dir2 = "dir1/dir2"; + bucket.createDirectory(dir2); + createSnapshot(testVolumeName, testBucketName, snap1); + bucket.deleteDirectory(dir1, true); + GenericTestUtils.waitFor(() -> { + try { + return getNumDirsInDeletedTable() == 0; + } catch (RocksDatabaseException | CodecException e) { + fail("Exception occurred while waiting for deletion" + e.getMessage()); + return false; + } + }, 100, 20000); + String snap2 = "snap2"; + createSnapshot(testVolumeName, testBucketName, snap2); + SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2); + List diffEntries = + Lists.newArrayList(SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.DELETE, dir1)); + if (disableNativeDiff) { + diffEntries.add(SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.DELETE, dir2)); + } + assertEquals(diff.getDiffList(), diffEntries); + stopKeyManager(); + } + + private int getNumDirsInDeletedTable() throws RocksDatabaseException, CodecException { + int numDirs = 0; + Table directoryTable = ozoneManager.getMetadataManager().getDirectoryTable(); + try (Table.KeyValueIterator it = directoryTable.iterator()) { + while (it.hasNext()) { + numDirs++; + it.next(); + } + return numDirs; + } + } + private OzoneObj buildKeyObj(OzoneBucket bucket, String key) { return OzoneObjInfo.Builder.newBuilder() .setResType(OzoneObj.ResourceType.KEY) From b44daed751e8a1bfc2f30f90f9b947d4d208510e Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Fri, 19 Dec 2025 21:53:01 +0530 Subject: [PATCH 13/13] add snapshot diff testcase --- .../ozone/om/snapshot/TestOmSnapshot.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index 0fed12e14d0f..fce3fae9a92c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -1111,9 +1111,11 @@ public void testSnapDiffWithDirectoryDeleteAfterDDSProcessing() throws Exception bucket.createDirectory(dir2); createSnapshot(testVolumeName, testBucketName, snap1); bucket.deleteDirectory(dir1, true); + // assert that dir2 exists + assertTrue(dirExists("dir2")); GenericTestUtils.waitFor(() -> { try { - return getNumDirsInDeletedTable() == 0; + return !dirExists("dir2"); } catch (RocksDatabaseException | CodecException e) { fail("Exception occurred while waiting for deletion" + e.getMessage()); return false; @@ -1131,15 +1133,19 @@ public void testSnapDiffWithDirectoryDeleteAfterDDSProcessing() throws Exception stopKeyManager(); } - private int getNumDirsInDeletedTable() throws RocksDatabaseException, CodecException { - int numDirs = 0; - Table directoryTable = ozoneManager.getMetadataManager().getDirectoryTable(); - try (Table.KeyValueIterator it = directoryTable.iterator()) { + private boolean dirExists(String dirName) throws RocksDatabaseException, + CodecException { + Table directoryTable = ozoneManager + .getMetadataManager().getDirectoryTable(); + try (Table.KeyValueIterator it = directoryTable + .iterator()) { while (it.hasNext()) { - numDirs++; - it.next(); + String name = it.next().getValue().getName(); + if (name.equals(dirName)) { + return true; + } } - return numDirs; + return false; } }