From afa8b6ca2485df3147c5146b564ab10fcbf5a266 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 20 Nov 2025 11:52:50 +0800 Subject: [PATCH 1/9] Recognizes expectedDataGeneration = -1L as a sentinel for "create if not exists" (If-None-Match) semantics. --- .../hadoop/ozone/client/OzoneBucket.java | 4 +-- .../hadoop/ozone/client/rpc/RpcClient.java | 4 +-- .../ozone/om/exceptions/OMException.java | 6 +++-- .../om/request/key/OMKeyCommitRequest.java | 26 +++++++++++++------ .../om/request/key/OMKeyCreateRequest.java | 25 ++++++++++++------ 5 files changed, 43 insertions(+), 22 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index b9649f09bb15..64a9a9317053 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -1029,8 +1029,8 @@ public List listStatus(String keyName, boolean recursive, * * @param prefix Optional string to filter for the selected keys. */ - public OzoneMultipartUploadList listMultipartUploads(String prefix, - String keyMarker, String uploadIdMarker, int maxUploads) + public OzoneMultipartUploadList listMultipartUploads(String prefix, + String keyMarker, String uploadIdMarker, int maxUploads) throws IOException { return proxy.listMultipartUploads(volumeName, getName(), prefix, keyMarker, uploadIdMarker, maxUploads); } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 4f6ddd76bafd..1c762f7a0367 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -683,7 +683,7 @@ public void createBucket( builder.setDefaultReplicationConfig(defaultReplicationConfig); } - String replicationType = defaultReplicationConfig == null + String replicationType = defaultReplicationConfig == null ? "server-side default replication type" : defaultReplicationConfig.getType().toString(); @@ -1321,7 +1321,7 @@ public List listBuckets(String volumeName, String bucketPrefix, List buckets = ozoneManagerClient.listBuckets( volumeName, prevBucket, bucketPrefix, maxListResult, hasSnapshot); - return buckets.stream().map(bucket -> + return buckets.stream().map(bucket -> OzoneBucket.newBuilder(conf, this) .setVolumeName(bucket.getVolumeName()) .setName(bucket.getBucketName()) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 596eb1276560..7f47c56fbf2f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -128,6 +128,8 @@ public enum ResultCodes { KEY_NOT_FOUND, + KEY_GENERATION_MISMATCH, + INVALID_KEY_NAME, ACCESS_DENIED, @@ -208,7 +210,7 @@ public enum ResultCodes { USER_MISMATCH, // Error code when requested user name passed is different // from remote user. - INVALID_PART, // When part name is not found or not matching with partname + INVALID_PART, // When part name is not found or not matching with partname // in OM MPU partInfo. INVALID_PART_ORDER, // When list of parts mentioned to complete MPU are not @@ -267,7 +269,7 @@ public enum ResultCodes { UNAUTHORIZED, S3_SECRET_ALREADY_EXISTS, - + INVALID_PATH, TOO_MANY_BUCKETS, KEY_UNDER_LEASE_RECOVERY, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index a106903bce79..6fae1b2b7721 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.request.key; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_CLOSED; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_GENERATION_MISMATCH; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_UNDER_LEASE_RECOVERY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; @@ -616,14 +617,23 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map if (toCommit.getExpectedDataGeneration() != null) { // These values are not passed in the request keyArgs, so add them into the auditMap if they are present // in the open key entry. - auditMap.put(OzoneConsts.REWRITE_GENERATION, String.valueOf(toCommit.getExpectedDataGeneration())); - if (existing == null) { - throw new OMException("Atomic rewrite is not allowed for a new key", KEY_NOT_FOUND); - } - if (!toCommit.getExpectedDataGeneration().equals(existing.getUpdateID())) { - throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + - ") does not match the expected generation to rewrite (" + toCommit.getExpectedDataGeneration() + ")", - KEY_NOT_FOUND); + Long expectedGen = toCommit.getExpectedDataGeneration(); + auditMap.put(OzoneConsts.REWRITE_GENERATION, String.valueOf(expectedGen)); + + if (expectedGen == -1L) { + if (existing != null) { + throw new OMException("Key already exists", + OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + } else { + if (existing == null) { + throw new OMException("Atomic rewrite is not allowed for a new key", KEY_NOT_FOUND); + } + if (!toCommit.getExpectedDataGeneration().equals(existing.getUpdateID())) { + throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + + ") does not match the expected generation to rewrite (" + toCommit.getExpectedDataGeneration() + ")", + KEY_GENERATION_MISMATCH); + } } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 6040cb7ddf6d..c33575cd819a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -178,7 +178,7 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { KeyArgs.Builder finalNewKeyArgs = newKeyArgs; KeyArgs resolvedKeyArgs = - captureLatencyNs(perfMetrics.getCreateKeyResolveBucketAndAclCheckLatencyNs(), + captureLatencyNs(perfMetrics.getCreateKeyResolveBucketAndAclCheckLatencyNs(), () -> resolveBucketAndCheckKeyAcls(finalNewKeyArgs.build(), ozoneManager, IAccessAuthorizer.ACLType.CREATE)); newCreateKeyRequest = @@ -358,7 +358,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } else { perfMetrics.addCreateKeyFailureLatencyNs(createKeyLatency); } - + if (acquireLock) { mergeOmLockDetails(ozoneLockStrategy .releaseWriteLock(omMetadataManager, volumeName, @@ -460,12 +460,21 @@ public static OMRequest blockCreateKeyWithBucketLayoutFromOldClient( protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) throws OMException { if (keyArgs.hasExpectedDataGeneration()) { - // If a key does not exist, or if it exists but the updateID do not match, then fail this request. - if (dbKeyInfo == null) { - throw new OMException("Key not found during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); - } - if (dbKeyInfo.getUpdateID() != keyArgs.getExpectedDataGeneration()) { - throw new OMException("Generation mismatch during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + long expectedGen = keyArgs.getExpectedDataGeneration(); + // If expectedGen is -1, it means the key MUST NOT exist (If-None-Match) + if (expectedGen == -1L) { + if (dbKeyInfo != null) { + throw new OMException("Key already exists", + OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + } else { + // If a key does not exist, or if it exists but the updateID do not match, then fail this request. + if (dbKeyInfo == null) { + throw new OMException("Key not found during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + } + if (dbKeyInfo.getUpdateID() != expectedGen) { + throw new OMException("Generation mismatch during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + } } } } From 7868a862ff616154e4bd9ab52dada929e15042ec Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 25 Nov 2025 00:32:30 +0800 Subject: [PATCH 2/9] correct the new result code position --- .../org/apache/hadoop/ozone/om/exceptions/OMException.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 7f47c56fbf2f..c95cfc162ba2 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -128,8 +128,6 @@ public enum ResultCodes { KEY_NOT_FOUND, - KEY_GENERATION_MISMATCH, - INVALID_KEY_NAME, ACCESS_DENIED, @@ -277,5 +275,7 @@ public enum ResultCodes { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD, TOO_MANY_SNAPSHOTS, + + KEY_GENERATION_MISMATCH, } } From da492ce17eec5721c5dd1c13519bf61a62150879 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 3 Dec 2025 23:38:50 +0800 Subject: [PATCH 3/9] HDDS-14070. Update key generation mismatch handling in Ozone RPC client tests and protocol definition. --- .../apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java | 3 ++- .../interface-client/src/main/proto/OmClientProtocol.proto | 2 ++ .../hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java | 5 +++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 554b125ddbec..646ea2b618fd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -39,6 +39,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.MD5_HASH; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_GENERATION_MISMATCH; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME; @@ -1356,7 +1357,7 @@ void rewriteFailsDueToOutdatedGenerationAtCommit(BucketLayout layout) throws IOE keyInfo = ozoneManager.lookupKey(keyArgs); OMException e = assertThrows(OMException.class, out::close); - assertEquals(KEY_NOT_FOUND, e.getResult()); + assertEquals(KEY_GENERATION_MISMATCH, e.getResult()); assertThat(e).hasMessageContaining("does not match the expected generation to rewrite"); } finally { if (out != null) { diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index bdb3cc3cee35..06cbf60ea746 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -566,6 +566,8 @@ enum Status { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD = 97; TOO_MANY_SNAPSHOTS = 98; + + KEY_GENERATION_MISMATCH = 99; } /** diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index e8bd2b079416..dc09fcae2623 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.om.request.key; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_GENERATION_MISMATCH; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.assertj.core.api.Assertions.assertThat; @@ -262,7 +263,7 @@ public void testAtomicRewrite() throws Exception { closedKeyTable.put(getOzonePathKey(), invalidKeyInfo); // This should fail as the updateID ia zero and the open key has rewrite generation of 1. omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); - assertEquals(KEY_NOT_FOUND, omClientResponse.getOMResponse().getStatus()); + assertEquals(KEY_GENERATION_MISMATCH, omClientResponse.getOMResponse().getStatus()); omKeyInfoBuilder.setUpdateID(1L); OmKeyInfo closedKeyInfo = omKeyInfoBuilder.build(); @@ -458,7 +459,7 @@ private Map doKeyCommit(boolean isHSync, .collect(Collectors.toList()); String openKey = addKeyToOpenKeyTable(allocatedBlockList); String ozoneKey = getOzonePathKey(); - + OMClientResponse omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); assertEquals(OK, From 0d4dfb0a35e730ea172323f52cc9648ecba7dafa Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 4 Dec 2025 00:05:28 +0800 Subject: [PATCH 4/9] Introduce EXPECTED_GEN_CREATE_IF_NOT_EXISTS constant --- .../src/main/java/org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 2 +- .../hadoop/ozone/om/request/key/OMKeyCreateRequest.java | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 42ca3f97b3b0..3d588e7ec006 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -314,6 +314,7 @@ public final class OzoneConsts { public static final String TENANT = "tenant"; public static final String USER_PREFIX = "userPrefix"; public static final String REWRITE_GENERATION = "rewriteGeneration"; + public static final long EXPECTED_GEN_CREATE_IF_NOT_EXISTS = -1L; public static final String FROM_SNAPSHOT = "fromSnapshot"; public static final String TO_SNAPSHOT = "toSnapshot"; public static final String TOKEN = "token"; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index e4eff3366c70..7bf705c9f64b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -620,7 +620,7 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map Long expectedGen = toCommit.getExpectedDataGeneration(); auditMap.put(OzoneConsts.REWRITE_GENERATION, String.valueOf(expectedGen)); - if (expectedGen == -1L) { + if (expectedGen == OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS) { if (existing != null) { throw new OMException("Key already exists", OMException.ResultCodes.KEY_ALREADY_EXISTS); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index c33575cd819a..febdba85f46c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.utils.UniqueId; import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -461,8 +462,8 @@ protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) throws OMException { if (keyArgs.hasExpectedDataGeneration()) { long expectedGen = keyArgs.getExpectedDataGeneration(); - // If expectedGen is -1, it means the key MUST NOT exist (If-None-Match) - if (expectedGen == -1L) { + // If expectedGen is EXPECTED_GEN_CREATE_IF_NOT_EXISTS, it means the key MUST NOT exist (If-None-Match) + if (expectedGen == OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS) { if (dbKeyInfo != null) { throw new OMException("Key already exists", OMException.ResultCodes.KEY_ALREADY_EXISTS); From 4e9665c29bf31ccb74b2330279a7de8e7475b0b8 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 2 Jan 2026 15:50:56 +0800 Subject: [PATCH 5/9] revert the new KEY_GENERATION_MISMATCH error code --- .../org/apache/hadoop/ozone/om/exceptions/OMException.java | 2 -- .../apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java | 2 +- .../interface-client/src/main/proto/OmClientProtocol.proto | 2 -- .../hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 3 +-- .../hadoop/ozone/om/request/key/OMKeyCreateRequest.java | 3 ++- .../hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java | 4 ++-- 6 files changed, 6 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index c95cfc162ba2..2b5b5559f92b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -275,7 +275,5 @@ public enum ResultCodes { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD, TOO_MANY_SNAPSHOTS, - - KEY_GENERATION_MISMATCH, } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 1a6cb4a9b36c..6ab484e0fe61 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -40,7 +40,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.client.OzoneClientTestUtils.assertKeyContent; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_GENERATION_MISMATCH; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME; diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 06cbf60ea746..bdb3cc3cee35 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -566,8 +566,6 @@ enum Status { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD = 97; TOO_MANY_SNAPSHOTS = 98; - - KEY_GENERATION_MISMATCH = 99; } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index cb2aacbdd63b..4d2ce035e9d9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.om.request.key; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_CLOSED; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_GENERATION_MISMATCH; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_UNDER_LEASE_RECOVERY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; @@ -632,7 +631,7 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map if (!toCommit.getExpectedDataGeneration().equals(existing.getUpdateID())) { throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + ") does not match the expected generation to rewrite (" + toCommit.getExpectedDataGeneration() + ")", - KEY_GENERATION_MISMATCH); + KEY_NOT_FOUND); } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index cd4f502e2fd8..2a17df2c7fd2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -474,7 +474,8 @@ protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) throw new OMException("Key not found during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); } if (dbKeyInfo.getUpdateID() != expectedGen) { - throw new OMException("Generation mismatch during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + throw new OMException("Generation mismatch during expected rewrite", + OMException.ResultCodes.KEY_NOT_FOUND); } } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index cf099d9c2b45..3d0eedec2fb4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -17,7 +17,7 @@ package org.apache.hadoop.ozone.om.request.key; -import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_GENERATION_MISMATCH; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.assertj.core.api.Assertions.assertThat; @@ -261,7 +261,7 @@ public void testAtomicRewrite() throws Exception { closedKeyTable.put(getOzonePathKey(), invalidKeyInfo); // This should fail as the updateID ia zero and the open key has rewrite generation of 1. omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); - assertEquals(KEY_GENERATION_MISMATCH, omClientResponse.getOMResponse().getStatus()); + assertEquals(KEY_NOT_FOUND, omClientResponse.getOMResponse().getStatus()); omKeyInfoBuilder.setUpdateID(1L); OmKeyInfo closedKeyInfo = omKeyInfoBuilder.build(); From b5249f1d8a1499a319f5bc8daa20b608f17d64b0 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 2 Jan 2026 15:59:37 +0800 Subject: [PATCH 6/9] Add tests for atomic key creation and rewriting with expected generation handling - Implemented tests for `createKey` and `rewriteKey` methods to validate behavior when using the `EXPECTED_GEN_CREATE_IF_NOT_EXISTS` constant. - Added scenarios for key creation when the key is absent and when it already exists. - Enhanced the `rewriteFailsWhenKeyExists` test to cover cases for both committed and uncommitted keys. - Updated error handling to ensure correct responses for key existence checks. --- .../ozone/client/rpc/OzoneRpcClientTests.java | 57 ++++++++++++- .../request/key/TestOMKeyCommitRequest.java | 70 ++++++++++++++++ .../request/key/TestOMKeyCreateRequest.java | 84 +++++++++++++++++++ 3 files changed, 210 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 6ab484e0fe61..879db6f7edca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -35,6 +35,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.DEFAULT_OM_UPDATE_ID; import static org.apache.hadoop.ozone.OzoneConsts.ETAG; +import static org.apache.hadoop.ozone.OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS; import static org.apache.hadoop.ozone.OzoneConsts.GB; import static org.apache.hadoop.ozone.OzoneConsts.MD5_HASH; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; @@ -1358,7 +1359,7 @@ void rewriteFailsDueToOutdatedGenerationAtCommit(BucketLayout layout) throws IOE keyInfo = ozoneManager.lookupKey(keyArgs); OMException e = assertThrows(OMException.class, out::close); - assertEquals(KEY_GENERATION_MISMATCH, e.getResult()); + assertEquals(KEY_NOT_FOUND, e.getResult()); assertThat(e).hasMessageContaining("does not match the expected generation to rewrite"); } finally { if (out != null) { @@ -1371,6 +1372,54 @@ void rewriteFailsDueToOutdatedGenerationAtCommit(BucketLayout layout) throws IOE assertUnchanged(keyInfo, ozoneManager.lookupKey(keyArgs)); } + @ParameterizedTest + @EnumSource + void rewriteFailsWhenKeyExists(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails key1Details = createTestKey(bucket, "key1", "value".getBytes(UTF_8)); + OzoneOutputStream key2Out = openTestKey(bucket, "key2", "value"); + OzoneOutputStream key3Out = openTestKey(bucket, "key3", "value"); + + // Test 1: Rewrite with -1 fails when key is already committed + OMException e = assertThrows(OMException.class, () -> { + bucket.rewriteKey( + key1Details.getName(), + key1Details.getDataSize(), + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + key1Details.getMetadata()); + }); + + assertEquals(KEY_ALREADY_EXISTS, e.getResult()); + assertThat(e).hasMessageContaining("Key already exists"); + + // Test 2: Rewrite with -1 succeeds when key is open but not yet committed + assertDoesNotThrow(() -> { + bucket.rewriteKey("key2", + 1024, + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", "value")); + }); + key2Out.close(); + + // Test 3: After rewrite completes, attempting to rewrite again with -1 fails + key3Out.write("value".getBytes(UTF_8)); + key3Out.close(); + + e = assertThrows(OMException.class, () -> { + bucket.rewriteKey("key2", + 1024, + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", "value")); + }); + + assertEquals(KEY_ALREADY_EXISTS, e.getResult()); + assertThat(e).hasMessageContaining("Key already exists"); + } + @ParameterizedTest @EnumSource void cannotRewriteDeletedKey(BucketLayout layout) throws IOException { @@ -4386,6 +4435,12 @@ private void completeMultipartUpload(OzoneBucket bucket, String keyName, assertNotNull(omMultipartUploadCompleteInfo.getHash()); } + private OzoneOutputStream openTestKey(OzoneBucket bucket, String keyName, String keyValue) throws IOException { + return bucket.createKey(keyName, keyValue.getBytes(UTF_8).length, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", RandomStringUtils.secure().nextAscii(10))); + } + private OzoneKeyDetails createTestKey(OzoneBucket bucket) throws IOException { return createTestKey(bucket, getTestName(), UUID.randomUUID().toString()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index 3d0eedec2fb4..e1ba052b821e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -280,6 +280,76 @@ public void testAtomicRewrite() throws Exception { assertEquals(acls, committedKey.getAcls()); } + @Test + public void testAtomicCreateIfNotExistsCommitKeyAbsent() throws Exception { + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedDataGeneration(OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + assertNull(closedKeyTable.get(getOzonePathKey())); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(OK, omClientResponse.getOMResponse().getStatus()); + + OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); + assertNotNull(committedKey); + assertNull(committedKey.getExpectedDataGeneration()); + } + + @Test + public void testAtomicCreateIfNotExistsCommitKeyAlreadyExists() throws Exception { + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedDataGeneration(OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + + OmKeyInfo existingClosedKey = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())).build(); + closedKeyTable.put(getOzonePathKey(), existingClosedKey); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_ALREADY_EXISTS, omClientResponse.getOMResponse().getStatus()); + } + @Test public void testValidateAndUpdateCacheWithUncommittedBlocks() throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java index b4cf5f7cc142..ecdc21745f50 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java @@ -27,6 +27,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.addVolumeAndBucketToDB; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.createOmKeyInfo; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.NOT_A_FILE; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; @@ -134,6 +135,89 @@ public void preExecuteRejectsInvalidReplication() { assertEquals(OMException.ResultCodes.INVALID_REQUEST, e.getResult()); } + @ParameterizedTest + @MethodSource("data") + public void testCreateKeyExpectedGenCreateIfNotExistsKeyMissing( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute(createKeyRequest( + false, 0, 100L, replicationConfig, + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS)); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + + checkResponse(modifiedOmRequest, response, id, false, getBucketLayout()); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateKeyExpectedGenCreateIfNotExistsKeyAlreadyExists( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute(createKeyRequest( + false, 0, 100L, replicationConfig, + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS)); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig).setUpdateID(1L).build(); + omMetadataManager.getKeyTable(getBucketLayout()).put(getOzoneKey(), existingKeyInfo); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + String openKey = getOpenKey(id); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_ALREADY_EXISTS, response.getOMResponse().getStatus()); + + // As we got error, no entry should be created in openKeyTable. + OmKeyInfo openKeyInfo = + omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); + assertNull(openKeyInfo); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateKeyExpectedGenMismatchReturnsKeyGenerationMismatch( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + long expectedGen = 1L; + OMRequest modifiedOmRequest = doPreExecute(createKeyRequest( + false, 0, 100L, replicationConfig, expectedGen)); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig).setUpdateID(2L).build(); + omMetadataManager.getKeyTable(getBucketLayout()).put(getOzoneKey(), existingKeyInfo); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + String openKey = getOpenKey(id); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_NOT_FOUND, response.getOMResponse().getStatus()); + + // As we got error, no entry should be created in openKeyTable. + OmKeyInfo openKeyInfo = + omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); + assertNull(openKeyInfo); + } + @ParameterizedTest @MethodSource("data") public void testValidateAndUpdateCache( From a1898c042df76276b412c62e6bd9d7b53373bcb7 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 25 Mar 2026 22:18:42 +0800 Subject: [PATCH 7/9] Add ATOMIC_CREATE_IF_NOT_EXISTS version gate --- .../org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../hadoop/ozone/OzoneManagerVersion.java | 3 + .../hadoop/ozone/client/OzoneBucket.java | 2 +- .../ozone/client/protocol/ClientProtocol.java | 2 +- .../hadoop/ozone/client/rpc/RpcClient.java | 3 + ...ManagerProtocolClientSideTranslatorPB.java | 2 +- .../ozone/client/rpc/OzoneRpcClientTests.java | 84 +++++++++---------- .../om/request/key/OMKeyCommitRequest.java | 8 +- .../om/request/key/OMKeyCreateRequest.java | 8 +- 9 files changed, 62 insertions(+), 51 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 0e8ea9d677fb..151022823a12 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -318,6 +318,7 @@ public final class OzoneConsts { public static final String TENANT = "tenant"; public static final String USER_PREFIX = "userPrefix"; public static final String REWRITE_GENERATION = "rewriteGeneration"; + /** Sentinel generation used to request atomic create-if-not-exists(put if absent) semantics. */ public static final long EXPECTED_GEN_CREATE_IF_NOT_EXISTS = -1L; public static final String FROM_SNAPSHOT = "fromSnapshot"; public static final String TO_SNAPSHOT = "toSnapshot"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index 41cf8ab28560..7d3f8629f0eb 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -54,6 +54,9 @@ public enum OzoneManagerVersion implements ComponentVersion { S3_LIST_MULTIPART_UPLOADS_PAGINATION(11, "OzoneManager version that supports S3 list multipart uploads API with pagination"), + + ATOMIC_CREATE_IF_NOT_EXISTS(12, + "OzoneManager version that supports explicit create-if-not-exists key semantics"), FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 6aa8ec15b72c..09cd189e8e48 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -508,7 +508,7 @@ public OzoneOutputStream createKey(String key, long size, * * @param keyName Existing key to rewrite. This must exist in the bucket. * @param size The size of the new key - * @param existingKeyGeneration The generation of the existing key which is checked for changes at key create + * @param existingKeyGeneration The positive generation of the existing key which is checked for changes at key create * and commit time. * @param replicationConfig The replication configuration for the key to be rewritten. * @param metadata custom key value metadata diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index e3a575896347..24ddd782cd27 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -368,7 +368,7 @@ OzoneOutputStream createKey(String volumeName, String bucketName, * @param bucketName Name of the Bucket * @param keyName Existing key to rewrite. This must exist in the bucket. * @param size The size of the new key - * @param existingKeyGeneration The generation of the existing key which is checked for changes at key create + * @param existingKeyGeneration The positive generation of the existing key which is checked for changes at key create * and commit time. * @param replicationConfig The replication configuration for the key to be rewritten. * @param metadata custom key value metadata diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 5a35d3a20d55..641a63a28f93 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1408,6 +1408,9 @@ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) { throw new IOException("OzoneManager does not support atomic key rewrite."); } + Preconditions.checkArgument(existingKeyGeneration > 0, + "existingKeyGeneration must be positive, but was %s", + existingKeyGeneration); createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 6960c11aaaa7..20f0babab82b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -1255,7 +1255,7 @@ public String createSnapshot(String volumeName, if (!StringUtils.isBlank(snapshotName)) { requestBuilder.setSnapshotName(snapshotName); } - + final OMRequest omRequest = createOMRequest(Type.CreateSnapshot) .setCreateSnapshotRequest(requestBuilder) .build(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 80da700fdd8d..fa2348127a48 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -1439,50 +1439,22 @@ void rewriteFailsDueToOutdatedGenerationAtCommit(BucketLayout layout) throws IOE @ParameterizedTest @EnumSource - void rewriteFailsWhenKeyExists(BucketLayout layout) throws IOException { + void rewriteRejectsNonPositiveGeneration(BucketLayout layout) + throws IOException { checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); OzoneBucket bucket = createBucket(layout); OzoneKeyDetails key1Details = createTestKey(bucket, "key1", "value".getBytes(UTF_8)); - OzoneOutputStream key2Out = openTestKey(bucket, "key2", "value"); - OzoneOutputStream key3Out = openTestKey(bucket, "key3", "value"); - - // Test 1: Rewrite with -1 fails when key is already committed - OMException e = assertThrows(OMException.class, () -> { - bucket.rewriteKey( - key1Details.getName(), - key1Details.getDataSize(), - EXPECTED_GEN_CREATE_IF_NOT_EXISTS, - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), - key1Details.getMetadata()); - }); - - assertEquals(KEY_ALREADY_EXISTS, e.getResult()); - assertThat(e).hasMessageContaining("Key already exists"); - - // Test 2: Rewrite with -1 succeeds when key is open but not yet committed - assertDoesNotThrow(() -> { - bucket.rewriteKey("key2", - 1024, - EXPECTED_GEN_CREATE_IF_NOT_EXISTS, - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), - singletonMap("key", "value")); - }); - key2Out.close(); - - // Test 3: After rewrite completes, attempting to rewrite again with -1 fails - key3Out.write("value".getBytes(UTF_8)); - key3Out.close(); - - e = assertThrows(OMException.class, () -> { - bucket.rewriteKey("key2", - 1024, - EXPECTED_GEN_CREATE_IF_NOT_EXISTS, - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), - singletonMap("key", "value")); - }); + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> { + bucket.rewriteKey("key2", + 1024, + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", "value")); + }); - assertEquals(KEY_ALREADY_EXISTS, e.getResult()); - assertThat(e).hasMessageContaining("Key already exists"); + assertThat(e).hasMessageContaining("existingKeyGeneration must be positive"); + assertKeyContent(bucket, key1Details.getName(), "value".getBytes(UTF_8)); } @ParameterizedTest @@ -4500,12 +4472,6 @@ private void completeMultipartUpload(OzoneBucket bucket, String keyName, assertNotNull(omMultipartUploadCompleteInfo.getHash()); } - private OzoneOutputStream openTestKey(OzoneBucket bucket, String keyName, String keyValue) throws IOException { - return bucket.createKey(keyName, keyValue.getBytes(UTF_8).length, - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), - singletonMap("key", RandomStringUtils.secure().nextAscii(10))); - } - private OzoneKeyDetails createTestKey(OzoneBucket bucket) throws IOException { return createTestKey(bucket, getTestName(), UUID.randomUUID().toString()); } @@ -4530,6 +4496,32 @@ private OzoneKeyDetails createTestKey( return key; } + private OzoneKeyDetails createTestKeyWithETag(OzoneBucket bucket) + throws IOException { + String keyName = getTestName(); + byte[] bytes = UUID.randomUUID().toString().getBytes(UTF_8); + RatisReplicationConfig replication = + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE); + Map metadata = metadataWithETag( + singletonMap("key", RandomStringUtils.secure().nextAscii(10)), bytes); + try (OzoneOutputStream out = bucket.createKey(keyName, bytes.length, + replication, metadata)) { + out.write(bytes); + } + OzoneKeyDetails key = bucket.getKey(keyName); + assertNotNull(key); + assertEquals(keyName, key.getName()); + assertEquals(DigestUtils.md5Hex(bytes), key.getMetadata().get(ETAG)); + return key; + } + + private static Map metadataWithETag( + Map metadata, byte[] data) { + Map metadataWithETag = new HashMap<>(metadata); + metadataWithETag.put(ETAG, DigestUtils.md5Hex(data)); + return metadataWithETag; + } + private void assertKeyRenamedEx(OzoneBucket bucket, String keyName) { OMException oe = assertThrows(OMException.class, () -> bucket.getKey(keyName)); assertEquals(KEY_NOT_FOUND, oe.getResult()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 4d2ce035e9d9..befa0031b954 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -94,7 +94,13 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { KeyArgs keyArgs = commitKeyRequest.getKeyArgs(); if (keyArgs.hasExpectedDataGeneration()) { - ozoneManager.checkFeatureEnabled(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + if (keyArgs.getExpectedDataGeneration() + == OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS) { + ozoneManager.checkFeatureEnabled( + OzoneManagerVersion.ATOMIC_CREATE_IF_NOT_EXISTS); + } else { + ozoneManager.checkFeatureEnabled(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + } } if (ozoneManager.getConfig().isKeyNameCharacterCheckEnabled()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 2581195adde4..1298ff7426fb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -96,7 +96,13 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { final OMPerformanceMetrics perfMetrics = ozoneManager.getPerfMetrics(); if (keyArgs.hasExpectedDataGeneration()) { - ozoneManager.checkFeatureEnabled(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + if (keyArgs.getExpectedDataGeneration() + == OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS) { + ozoneManager.checkFeatureEnabled( + OzoneManagerVersion.ATOMIC_CREATE_IF_NOT_EXISTS); + } else { + ozoneManager.checkFeatureEnabled(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + } } OmUtils.verifyKeyNameWithSnapshotReservedWord(keyArgs.getKeyName()); From e5f5e3d3408605daf1f6f1844a9293e02c8d5be1 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 25 Mar 2026 22:36:06 +0800 Subject: [PATCH 8/9] suggestion --- .../apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java | 1 - .../hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index fa2348127a48..ff602c8d90ac 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -42,7 +42,6 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.client.OzoneClientTestUtils.assertKeyContent; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index befa0031b954..492d698db997 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -634,9 +634,9 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map if (existing == null) { throw new OMException("Atomic rewrite is not allowed for a new key", KEY_NOT_FOUND); } - if (!toCommit.getExpectedDataGeneration().equals(existing.getUpdateID())) { + if (expectedGen != existing.getUpdateID()) { throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + - ") does not match the expected generation to rewrite (" + toCommit.getExpectedDataGeneration() + ")", + ") does not match the expected generation to rewrite (" + expectedGen + ")", KEY_NOT_FOUND); } } From 86489b6deb1c6a4f25961e725e305be535b97a56 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 25 Mar 2026 22:43:08 +0800 Subject: [PATCH 9/9] fix pmd --- .../ozone/client/rpc/OzoneRpcClientTests.java | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index ff602c8d90ac..378316c6b15e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -4495,32 +4495,6 @@ private OzoneKeyDetails createTestKey( return key; } - private OzoneKeyDetails createTestKeyWithETag(OzoneBucket bucket) - throws IOException { - String keyName = getTestName(); - byte[] bytes = UUID.randomUUID().toString().getBytes(UTF_8); - RatisReplicationConfig replication = - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE); - Map metadata = metadataWithETag( - singletonMap("key", RandomStringUtils.secure().nextAscii(10)), bytes); - try (OzoneOutputStream out = bucket.createKey(keyName, bytes.length, - replication, metadata)) { - out.write(bytes); - } - OzoneKeyDetails key = bucket.getKey(keyName); - assertNotNull(key); - assertEquals(keyName, key.getName()); - assertEquals(DigestUtils.md5Hex(bytes), key.getMetadata().get(ETAG)); - return key; - } - - private static Map metadataWithETag( - Map metadata, byte[] data) { - Map metadataWithETag = new HashMap<>(metadata); - metadataWithETag.put(ETAG, DigestUtils.md5Hex(data)); - return metadataWithETag; - } - private void assertKeyRenamedEx(OzoneBucket bucket, String keyName) { OMException oe = assertThrows(OMException.class, () -> bucket.getKey(keyName)); assertEquals(KEY_NOT_FOUND, oe.getResult());