From a342fe2d6bb6f90c301a18a0b27aaf6caaadc139 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 12 Nov 2025 12:02:20 +0800 Subject: [PATCH 01/10] HDDS-13490. Implement KeyInfoProtoLight-based PartKeyInfoMap in MultipartInfoInsightHandler --- .../ozone/s3/endpoint/BucketEndpoint.java | 58 +++++++++++++++++-- .../endpoint/ListMultipartUploadsResult.java | 48 ++++++++++++++- 2 files changed, 98 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index c808f0cce761..c4d69eb5304e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -146,7 +146,8 @@ public Response get( if (uploads != null) { s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; - return listMultipartUploads(bucketName, prefix, keyMarker, uploadIdMarker, maxUploads); + return listMultipartUploads(bucketName, prefix, delimiter, encodingType, + keyMarker, uploadIdMarker, maxUploads); } maxKeys = validateMaxKeys(maxKeys); @@ -351,6 +352,8 @@ public Response put(@PathParam("bucket") String bucketName, public Response listMultipartUploads( String bucketName, String prefix, + String delimiter, + String encodingType, String keyMarker, String uploadIdMarker, int maxUploads) @@ -363,6 +366,15 @@ public Response listMultipartUploads( maxUploads = Math.min(maxUploads, 1000); } + // The valid encodingType Values is "url" + if (encodingType != null && !encodingType.equals(ENCODING_TYPE)) { + throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, encodingType); + } + + if (prefix == null) { + prefix = ""; + } + long startNanos = Time.monotonicNowNanos(); S3GAction s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; @@ -378,18 +390,54 @@ public Response listMultipartUploads( result.setKeyMarker(keyMarker); result.setUploadIdMarker(uploadIdMarker); result.setNextKeyMarker(ozoneMultipartUploadList.getNextKeyMarker()); - result.setPrefix(prefix); + result.setPrefix(EncodingTypeObject.createNullable(prefix, encodingType)); + result.setDelimiter(EncodingTypeObject.createNullable(delimiter, encodingType)); + result.setEncodingType(encodingType); result.setNextUploadIdMarker(ozoneMultipartUploadList.getNextUploadIdMarker()); result.setMaxUploads(maxUploads); result.setTruncated(ozoneMultipartUploadList.isTruncated()); - ozoneMultipartUploadList.getUploads().forEach(upload -> result.addUpload( - new ListMultipartUploadsResult.Upload( + String prevDir = null; + for (org.apache.hadoop.ozone.client.OzoneMultipartUpload upload : + ozoneMultipartUploadList.getUploads()) { + String keyName = upload.getKeyName(); + if (bucket.getBucketLayout().isFileSystemOptimized() && + StringUtils.isNotEmpty(prefix) && + !keyName.startsWith(prefix)) { + continue; + } + String relativeKeyName = keyName.substring(prefix.length()); + + if (!StringUtils.isEmpty(delimiter)) { + int depth = StringUtils.countMatches(relativeKeyName, delimiter); + if (depth > 0) { + String dirName = relativeKeyName.substring(0, relativeKeyName.indexOf(delimiter)); + if (!dirName.equals(prevDir)) { + result.addPrefix(EncodingTypeObject.createNullable( + prefix + dirName + delimiter, encodingType)); + prevDir = dirName; + } + } else if (relativeKeyName.endsWith(delimiter)) { + result.addPrefix(EncodingTypeObject.createNullable( + prefix + relativeKeyName, encodingType)); + } else { + result.addUpload(new ListMultipartUploadsResult.Upload( + upload.getKeyName(), + upload.getUploadId(), + upload.getCreationTime(), + S3StorageType.fromReplicationConfig(upload.getReplicationConfig()) + )); + } + } else { + result.addUpload(new ListMultipartUploadsResult.Upload( upload.getKeyName(), upload.getUploadId(), upload.getCreationTime(), S3StorageType.fromReplicationConfig(upload.getReplicationConfig()) - ))); + )); + } + } + AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction, getAuditParameters())); getMetrics().updateListMultipartUploadsSuccessStats(startNanos); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java index 98801a520e96..efd016c661bf 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java @@ -25,7 +25,10 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import org.apache.hadoop.ozone.s3.commontypes.CommonPrefix; +import org.apache.hadoop.ozone.s3.commontypes.EncodingTypeObject; import org.apache.hadoop.ozone.s3.commontypes.IsoDateAdapter; +import org.apache.hadoop.ozone.s3.commontypes.ObjectKeyNameAdapter; import org.apache.hadoop.ozone.s3.util.S3Consts; import org.apache.hadoop.ozone.s3.util.S3StorageType; @@ -49,8 +52,16 @@ public class ListMultipartUploadsResult { @XmlElement(name = "NextKeyMarker") private String nextKeyMarker; + @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) @XmlElement(name = "Prefix") - private String prefix; + private EncodingTypeObject prefix; + + @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) + @XmlElement(name = "Delimiter") + private EncodingTypeObject delimiter; + + @XmlElement(name = "EncodingType") + private String encodingType; @XmlElement(name = "NextUploadIdMarker") private String nextUploadIdMarker; @@ -64,6 +75,9 @@ public class ListMultipartUploadsResult { @XmlElement(name = "Upload") private List uploads = new ArrayList<>(); + @XmlElement(name = "CommonPrefixes") + private List commonPrefixes = new ArrayList<>(); + public String getBucket() { return bucket; } @@ -96,14 +110,30 @@ public void setNextKeyMarker(String nextKeyMarker) { this.nextKeyMarker = nextKeyMarker; } - public String getPrefix() { + public EncodingTypeObject getPrefix() { return prefix; } - public void setPrefix(String prefix) { + public void setPrefix(EncodingTypeObject prefix) { this.prefix = prefix; } + public EncodingTypeObject getDelimiter() { + return delimiter; + } + + public void setDelimiter(EncodingTypeObject delimiter) { + this.delimiter = delimiter; + } + + public String getEncodingType() { + return encodingType; + } + + public void setEncodingType(String encodingType) { + this.encodingType = encodingType; + } + public String getNextUploadIdMarker() { return nextUploadIdMarker; } @@ -141,6 +171,18 @@ public void addUpload(Upload upload) { this.uploads.add(upload); } + public List getCommonPrefixes() { + return commonPrefixes; + } + + public void setCommonPrefixes(List commonPrefixes) { + this.commonPrefixes = commonPrefixes; + } + + public void addPrefix(EncodingTypeObject prefix) { + commonPrefixes.add(new CommonPrefix(prefix)); + } + /** * Upload information. */ From 8a318626aff58f8c83e650fd41cc6ce47047f88b Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 12 Nov 2025 12:28:48 +0800 Subject: [PATCH 02/10] fix checkstyle --- .../org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index c4d69eb5304e..2eacc9202180 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -63,6 +63,7 @@ import org.apache.hadoop.ozone.audit.S3GAction; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneKey; +import org.apache.hadoop.ozone.client.OzoneMultipartUpload; import org.apache.hadoop.ozone.client.OzoneMultipartUploadList; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -398,8 +399,7 @@ public Response listMultipartUploads( result.setTruncated(ozoneMultipartUploadList.isTruncated()); String prevDir = null; - for (org.apache.hadoop.ozone.client.OzoneMultipartUpload upload : - ozoneMultipartUploadList.getUploads()) { + for (OzoneMultipartUpload upload : ozoneMultipartUploadList.getUploads()) { String keyName = upload.getKeyName(); if (bucket.getBucketLayout().isFileSystemOptimized() && StringUtils.isNotEmpty(prefix) && From 4d220d36e398e14b22dbe97811cf1af397f08d63 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 12 Nov 2025 13:19:51 +0800 Subject: [PATCH 03/10] Fix TestPermissionCheck: update listMultipartUploads call to match new signature --- .../hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java | 4 ++-- .../apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java index efd016c661bf..7e262491f474 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java @@ -179,8 +179,8 @@ public void setCommonPrefixes(List commonPrefixes) { this.commonPrefixes = commonPrefixes; } - public void addPrefix(EncodingTypeObject prefix) { - commonPrefixes.add(new CommonPrefix(prefix)); + public void addPrefix(EncodingTypeObject relativeKeyName) { + commonPrefixes.add(new CommonPrefix(relativeKeyName)); } /** diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index 81f6853bf73f..27cb024e4f56 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -154,7 +154,7 @@ public void testListMultiUpload() throws IOException { .setClient(client) .build(); OS3Exception e = assertThrows(OS3Exception.class, () -> - bucketEndpoint.listMultipartUploads("bucketName", "prefix", "", "", 10)); + bucketEndpoint.listMultipartUploads("bucketName", "prefix", null, null, "", "", 10)); assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); } From c7f70f8b9a2ecf94676ceea7c88cf441e5deb0f4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 14 Nov 2025 16:42:08 +0800 Subject: [PATCH 04/10] fix consolidate duplicate conditional fragments and modify KeyMarker & NextKeyMarker --- .../ozone/s3/endpoint/BucketEndpoint.java | 19 +++++++++---------- .../endpoint/ListMultipartUploadsResult.java | 14 ++++++++------ 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 2eacc9202180..0965e489eb55 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -388,9 +388,10 @@ public Response listMultipartUploads( ListMultipartUploadsResult result = new ListMultipartUploadsResult(); result.setBucket(bucketName); - result.setKeyMarker(keyMarker); + result.setKeyMarker(EncodingTypeObject.createNullable(keyMarker, encodingType)); result.setUploadIdMarker(uploadIdMarker); - result.setNextKeyMarker(ozoneMultipartUploadList.getNextKeyMarker()); + result.setNextKeyMarker(EncodingTypeObject.createNullable( + ozoneMultipartUploadList.getNextKeyMarker(), encodingType)); result.setPrefix(EncodingTypeObject.createNullable(prefix, encodingType)); result.setDelimiter(EncodingTypeObject.createNullable(delimiter, encodingType)); result.setEncodingType(encodingType); @@ -408,6 +409,7 @@ public Response listMultipartUploads( } String relativeKeyName = keyName.substring(prefix.length()); + boolean addedAsPrefix = false; if (!StringUtils.isEmpty(delimiter)) { int depth = StringUtils.countMatches(relativeKeyName, delimiter); if (depth > 0) { @@ -416,19 +418,16 @@ public Response listMultipartUploads( result.addPrefix(EncodingTypeObject.createNullable( prefix + dirName + delimiter, encodingType)); prevDir = dirName; + addedAsPrefix = true; } } else if (relativeKeyName.endsWith(delimiter)) { result.addPrefix(EncodingTypeObject.createNullable( prefix + relativeKeyName, encodingType)); - } else { - result.addUpload(new ListMultipartUploadsResult.Upload( - upload.getKeyName(), - upload.getUploadId(), - upload.getCreationTime(), - S3StorageType.fromReplicationConfig(upload.getReplicationConfig()) - )); + addedAsPrefix = true; } - } else { + } + + if (!addedAsPrefix) { result.addUpload(new ListMultipartUploadsResult.Upload( upload.getKeyName(), upload.getUploadId(), diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java index 7e262491f474..b7f36107c110 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java @@ -43,14 +43,16 @@ public class ListMultipartUploadsResult { @XmlElement(name = "Bucket") private String bucket; + @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) @XmlElement(name = "KeyMarker") - private String keyMarker; + private EncodingTypeObject keyMarker; @XmlElement(name = "UploadIdMarker") private String uploadIdMarker; + @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) @XmlElement(name = "NextKeyMarker") - private String nextKeyMarker; + private EncodingTypeObject nextKeyMarker; @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) @XmlElement(name = "Prefix") @@ -86,11 +88,11 @@ public void setBucket(String bucket) { this.bucket = bucket; } - public String getKeyMarker() { + public EncodingTypeObject getKeyMarker() { return keyMarker; } - public void setKeyMarker(String keyMarker) { + public void setKeyMarker(EncodingTypeObject keyMarker) { this.keyMarker = keyMarker; } @@ -102,11 +104,11 @@ public void setUploadIdMarker(String uploadIdMarker) { this.uploadIdMarker = uploadIdMarker; } - public String getNextKeyMarker() { + public EncodingTypeObject getNextKeyMarker() { return nextKeyMarker; } - public void setNextKeyMarker(String nextKeyMarker) { + public void setNextKeyMarker(EncodingTypeObject nextKeyMarker) { this.nextKeyMarker = nextKeyMarker; } From 55e3aa84001112c6b3355efeb40e0ae89e0e0200 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 15 Nov 2025 21:59:11 +0800 Subject: [PATCH 05/10] move repeat function --- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 0965e489eb55..7e8e8aa94259 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -145,6 +145,10 @@ public Response get( return Response.ok(result, MediaType.APPLICATION_XML_TYPE).build(); } + if (prefix == null) { + prefix = ""; + } + if (uploads != null) { s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; return listMultipartUploads(bucketName, prefix, delimiter, encodingType, @@ -153,10 +157,6 @@ public Response get( maxKeys = validateMaxKeys(maxKeys); - if (prefix == null) { - prefix = ""; - } - // Assign marker to startAfter. for the compatibility of aws api v1 if (startAfter == null && marker != null) { startAfter = marker; @@ -372,10 +372,6 @@ public Response listMultipartUploads( throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, encodingType); } - if (prefix == null) { - prefix = ""; - } - long startNanos = Time.monotonicNowNanos(); S3GAction s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; From e019d5e4e103f67bf5fdaeae80becce99f9875c9 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 18 Nov 2025 10:02:27 +0800 Subject: [PATCH 06/10] modify key in upload to encodinf type --- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 2 +- .../ozone/s3/endpoint/ListMultipartUploadsResult.java | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 7e8e8aa94259..ebcd5bebd201 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -425,7 +425,7 @@ public Response listMultipartUploads( if (!addedAsPrefix) { result.addUpload(new ListMultipartUploadsResult.Upload( - upload.getKeyName(), + EncodingTypeObject.createNullable(upload.getKeyName(), encodingType), upload.getUploadId(), upload.getCreationTime(), S3StorageType.fromReplicationConfig(upload.getReplicationConfig()) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java index b7f36107c110..7bcc8a815e7e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java @@ -192,8 +192,9 @@ public void addPrefix(EncodingTypeObject relativeKeyName) { @XmlRootElement(name = "Upload") public static class Upload { + @XmlJavaTypeAdapter(ObjectKeyNameAdapter.class) @XmlElement(name = "Key") - private String key; + private EncodingTypeObject key; @XmlElement(name = "UploadId") private String uploadId; @@ -214,13 +215,13 @@ public static class Upload { public Upload() { } - public Upload(String key, String uploadId, Instant initiated) { + public Upload(EncodingTypeObject key, String uploadId, Instant initiated) { this.key = key; this.uploadId = uploadId; this.initiated = initiated; } - public Upload(String key, String uploadId, Instant initiated, + public Upload(EncodingTypeObject key, String uploadId, Instant initiated, S3StorageType storageClass) { this.key = key; this.uploadId = uploadId; @@ -228,11 +229,11 @@ public Upload(String key, String uploadId, Instant initiated, this.storageClass = storageClass.toString(); } - public String getKey() { + public EncodingTypeObject getKey() { return key; } - public void setKey(String key) { + public void setKey(EncodingTypeObject key) { this.key = key; } From 72e13d7b47315d13c3dd698b2b30449728f0532e Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 21 Nov 2025 17:51:12 +0800 Subject: [PATCH 07/10] WIP-chang function name make it clear --- .../hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java index 7bcc8a815e7e..aadecf10b356 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java @@ -181,7 +181,7 @@ public void setCommonPrefixes(List commonPrefixes) { this.commonPrefixes = commonPrefixes; } - public void addPrefix(EncodingTypeObject relativeKeyName) { + public void addCommonPrefix(EncodingTypeObject relativeKeyName) { commonPrefixes.add(new CommonPrefix(relativeKeyName)); } From 6ee325356d1ddb297d8283899a76ff2d5d423e4a Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 23 Nov 2025 11:46:30 +0800 Subject: [PATCH 08/10] add modify --- .../ozone/s3/endpoint/BucketEndpoint.java | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index ebcd5bebd201..b8eb3bff1157 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -395,34 +395,53 @@ public Response listMultipartUploads( result.setMaxUploads(maxUploads); result.setTruncated(ozoneMultipartUploadList.isTruncated()); + final String normalizedPrefix = prefix == null ? "" : prefix; + // track the previous directory name to avoid duplicate CommonPrefixes entries String prevDir = null; for (OzoneMultipartUpload upload : ozoneMultipartUploadList.getUploads()) { String keyName = upload.getKeyName(); + // filter out keys that don't match the prefix in FS-optimized buckets if (bucket.getBucketLayout().isFileSystemOptimized() && - StringUtils.isNotEmpty(prefix) && - !keyName.startsWith(prefix)) { + StringUtils.isNotEmpty(normalizedPrefix) && + !keyName.startsWith(normalizedPrefix)) { + continue; + } + // skip keys shorter than prefix to avoid substring errors + if (keyName.length() < normalizedPrefix.length()) { continue; } - String relativeKeyName = keyName.substring(prefix.length()); + // relative key name after removing the prefix + String relativeKeyName = keyName.substring(normalizedPrefix.length()); + // Track whether this upload was added as a CommonPrefix boolean addedAsPrefix = false; - if (!StringUtils.isEmpty(delimiter)) { + // Only extract common prefixes when delimiter is provided + if (StringUtils.isNotBlank(delimiter)) { + // collapse objects that share the same delimiter + // first segment after the prefix into a single CommonPrefixes entry. + // This matches AWS behavior where keys under the same "directory" are grouped. int depth = StringUtils.countMatches(relativeKeyName, delimiter); if (depth > 0) { - String dirName = relativeKeyName.substring(0, relativeKeyName.indexOf(delimiter)); + // Key has delimiter(s): extract the first directory segment + int delimiterIndex = relativeKeyName.indexOf(delimiter); + String dirName = relativeKeyName.substring(0, delimiterIndex); + // Only add CommonPrefix if this directory hasn't been added yet (deduplication) if (!dirName.equals(prevDir)) { - result.addPrefix(EncodingTypeObject.createNullable( - prefix + dirName + delimiter, encodingType)); + result.addCommonPrefix(EncodingTypeObject.createNullable( + normalizedPrefix + dirName + delimiter, encodingType)); prevDir = dirName; - addedAsPrefix = true; } + addedAsPrefix = true; } else if (relativeKeyName.endsWith(delimiter)) { - result.addPrefix(EncodingTypeObject.createNullable( - prefix + relativeKeyName, encodingType)); + // Key itself ends with delimiter (represents a "directory" placeholder) + result.addCommonPrefix(EncodingTypeObject.createNullable( + normalizedPrefix + relativeKeyName, encodingType)); addedAsPrefix = true; } } - + + // This ensures all uploads are represented: either as individual Upload entries + // or as part of CommonPrefixes (when delimiter is used). if (!addedAsPrefix) { result.addUpload(new ListMultipartUploadsResult.Upload( EncodingTypeObject.createNullable(upload.getKeyName(), encodingType), From d25c61ced736342df8120e2e8db64a08466b938f Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 27 Nov 2025 17:28:32 +0800 Subject: [PATCH 09/10] add test in bucketlist and pagination --- .../ozone/s3/endpoint/BucketEndpoint.java | 100 ++++++++++---- .../hadoop/ozone/client/OzoneBucketStub.java | 58 +++++++- .../ozone/s3/endpoint/TestBucketList.java | 130 ++++++++++++++++++ 3 files changed, 263 insertions(+), 25 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index b8eb3bff1157..cd797c1f9447 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -396,52 +396,84 @@ public Response listMultipartUploads( result.setTruncated(ozoneMultipartUploadList.isTruncated()); final String normalizedPrefix = prefix == null ? "" : prefix; - // track the previous directory name to avoid duplicate CommonPrefixes entries + // Track the previous directory name to avoid duplicate CommonPrefixes entries String prevDir = null; - for (OzoneMultipartUpload upload : ozoneMultipartUploadList.getUploads()) { + // Track the last key processed for pagination + String lastProcessedKey = null; + String lastProcessedUploadId = null; + // Count of items added to response (CommonPrefixes + Uploads) + int responseItemCount = 0; + + List pendingUploads = + ozoneMultipartUploadList.getUploads(); + int processedUploads = 0; + for (OzoneMultipartUpload upload : pendingUploads) { String keyName = upload.getKeyName(); - // filter out keys that don't match the prefix in FS-optimized buckets + + // Filter out keys that don't match the prefix in FS-optimized buckets if (bucket.getBucketLayout().isFileSystemOptimized() && StringUtils.isNotEmpty(normalizedPrefix) && !keyName.startsWith(normalizedPrefix)) { continue; } - // skip keys shorter than prefix to avoid substring errors + // Skip keys shorter than prefix to avoid substring errors if (keyName.length() < normalizedPrefix.length()) { continue; } - // relative key name after removing the prefix - String relativeKeyName = keyName.substring(normalizedPrefix.length()); - // Track whether this upload was added as a CommonPrefix - boolean addedAsPrefix = false; - // Only extract common prefixes when delimiter is provided + // Relative key name after removing the prefix + String relativeKeyName = keyName.substring(normalizedPrefix.length()); + String currentDirName = null; + boolean isDirectoryPlaceholder = false; if (StringUtils.isNotBlank(delimiter)) { - // collapse objects that share the same delimiter - // first segment after the prefix into a single CommonPrefixes entry. - // This matches AWS behavior where keys under the same "directory" are grouped. int depth = StringUtils.countMatches(relativeKeyName, delimiter); if (depth > 0) { - // Key has delimiter(s): extract the first directory segment int delimiterIndex = relativeKeyName.indexOf(delimiter); - String dirName = relativeKeyName.substring(0, delimiterIndex); - // Only add CommonPrefix if this directory hasn't been added yet (deduplication) - if (!dirName.equals(prevDir)) { - result.addCommonPrefix(EncodingTypeObject.createNullable( - normalizedPrefix + dirName + delimiter, encodingType)); - prevDir = dirName; - } - addedAsPrefix = true; + currentDirName = relativeKeyName.substring(0, delimiterIndex); } else if (relativeKeyName.endsWith(delimiter)) { - // Key itself ends with delimiter (represents a "directory" placeholder) + currentDirName = relativeKeyName.substring( + 0, relativeKeyName.length() - delimiter.length()); + isDirectoryPlaceholder = true; + } + } + + // If we've already returned maxUploads items, keep iterating only to drain + // entries that belong to the same prefix (so that the next page won't repeat them). + if (responseItemCount >= maxUploads) { + if (StringUtils.isNotBlank(delimiter) + && currentDirName != null + && currentDirName.equals(prevDir)) { + lastProcessedKey = keyName; + lastProcessedUploadId = upload.getUploadId(); + continue; + } + break; + } + + // Track whether this upload was added as a CommonPrefix + boolean addedAsPrefix = false; + + // When delimiter is provided, group keys by common prefix (AWS S3 behavior) + // Example: with delimiter="/", keys "dir1/file1" and "dir1/file2" + // are collapsed into CommonPrefix "dir1/" instead of individual Upload entries + if (StringUtils.isNotBlank(delimiter)) { + if (currentDirName != null && !currentDirName.equals(prevDir)) { + result.addCommonPrefix(EncodingTypeObject.createNullable( + normalizedPrefix + currentDirName + delimiter, encodingType)); + prevDir = currentDirName; + responseItemCount++; + addedAsPrefix = true; + } else if (isDirectoryPlaceholder) { result.addCommonPrefix(EncodingTypeObject.createNullable( normalizedPrefix + relativeKeyName, encodingType)); + responseItemCount++; + addedAsPrefix = true; + } else if (currentDirName != null) { addedAsPrefix = true; } } - // This ensures all uploads are represented: either as individual Upload entries - // or as part of CommonPrefixes (when delimiter is used). + // Add as individual Upload entry if not collapsed into CommonPrefix if (!addedAsPrefix) { result.addUpload(new ListMultipartUploadsResult.Upload( EncodingTypeObject.createNullable(upload.getKeyName(), encodingType), @@ -449,7 +481,27 @@ public Response listMultipartUploads( upload.getCreationTime(), S3StorageType.fromReplicationConfig(upload.getReplicationConfig()) )); + responseItemCount++; } + + // Track the last processed key for pagination + lastProcessedKey = keyName; + lastProcessedUploadId = upload.getUploadId(); + processedUploads++; + } + + boolean hasMoreUploads = + processedUploads < pendingUploads.size() + || ozoneMultipartUploadList.isTruncated(); + + // Override NextKeyMarker and NextUploadIdMarker if we stopped early due to maxUploads + if (responseItemCount >= maxUploads && lastProcessedKey != null + && hasMoreUploads) { + result.setNextKeyMarker(EncodingTypeObject.createNullable(lastProcessedKey, encodingType)); + result.setNextUploadIdMarker(lastProcessedUploadId); + result.setTruncated(true); + } else { + result.setTruncated(ozoneMultipartUploadList.isTruncated()); } AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction, diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 5d060c228c81..06dc7b3b6857 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -27,12 +27,14 @@ import java.nio.ByteBuffer; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.SortedMap; import java.util.TreeMap; import java.util.UUID; import java.util.stream.Collectors; @@ -575,7 +577,55 @@ public OzoneMultipartUploadPartListParts listParts(String key, @Override public List getAcls() throws IOException { - return (List)aclList.clone(); + return new ArrayList<>(aclList); + } + + @Override + public OzoneMultipartUploadList listMultipartUploads(String prefix, + String keyMarker, String uploadIdMarker, int maxUploads) + throws IOException { + String normalizedPrefix = prefix == null ? "" : prefix; + SortedMap sortedUploads = + new TreeMap<>(keyToMultipartUpload); + List uploads = new ArrayList<>(); + for (Map.Entry entry : sortedUploads.entrySet()) { + String keyName = entry.getKey(); + MultipartInfoStub uploadInfo = entry.getValue(); + + if (!keyName.startsWith(normalizedPrefix)) { + continue; + } + + if (StringUtils.isNotEmpty(keyMarker)) { + int cmp = keyName.compareTo(keyMarker); + if (cmp < 0) { + continue; + } + if (cmp == 0) { + if (StringUtils.isNotEmpty(uploadIdMarker)) { + if (uploadInfo.getUploadId().compareTo(uploadIdMarker) <= 0) { + continue; + } + } else { + continue; + } + } + } + + uploads.add(new OzoneMultipartUpload( + getVolumeName(), + getName(), + keyName, + uploadInfo.getUploadId(), + uploadInfo.getCreationTime(), + getReplicationConfig())); + } + + return new OzoneMultipartUploadList( + uploads, + null, + null, + false); } @Override @@ -764,12 +814,14 @@ private static class MultipartInfoStub { private final String uploadId; private final Map metadata; private final Map tags; + private final Instant creationTime; MultipartInfoStub(String uploadId, Map metadata, Map tags) { this.uploadId = uploadId; this.metadata = metadata; this.tags = tags; + this.creationTime = Instant.now(); } public String getUploadId() { @@ -783,6 +835,10 @@ public Map getMetadata() { public Map getTags() { return tags; } + + public Instant getCreationTime() { + return creationTime; + } } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java index f23f8f81b60d..c9ccbdbccb6f 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java @@ -625,4 +625,134 @@ private OzoneClient createClientWithKeys(String... keys) throws IOException { } return client; } + + private OzoneClient createClientWithMultipartUploads(String... keys) + throws IOException { + OzoneClient client = new OzoneClientStub(); + client.getObjectStore().createS3Bucket("b1"); + OzoneBucket bucket = client.getObjectStore().getS3Bucket("b1"); + for (String key : keys) { + bucket.initiateMultipartUpload(key); + } + return client; + } + + @Test + public void listMultipartUploadsWithDelimiter() throws OS3Exception, IOException { + BucketEndpoint getBucket = new BucketEndpoint(); + + OzoneClient ozoneClient = createClientWithMultipartUploads( + "dir1/file1", + "dir1/file2", + "dir2/file3", + "file4" + ); + + getBucket.setClient(ozoneClient); + getBucket.setRequestIdentifier(new RequestIdentifier()); + + ListMultipartUploadsResult result = + (ListMultipartUploadsResult) getBucket.get("b1", "/", null, null, 100, + "", null, null, "uploads", null, null, null, 1000).getEntity(); + + // With delimiter="/", should have 2 CommonPrefixes (dir1/, dir2/) and 1 Upload (file4) + assertEquals(2, result.getCommonPrefixes().size()); + assertEquals("dir1/", result.getCommonPrefixes().get(0).getPrefix().getName()); + assertEquals("dir2/", result.getCommonPrefixes().get(1).getPrefix().getName()); + assertEquals(1, result.getUploads().size()); + assertEquals("file4", result.getUploads().get(0).getKey().getName()); + } + + @Test + public void listMultipartUploadsWithDelimiterAndPrefix() throws OS3Exception, IOException { + BucketEndpoint getBucket = new BucketEndpoint(); + + OzoneClient ozoneClient = createClientWithMultipartUploads( + "test/dir1/file1", + "test/dir1/file2", + "test/dir2/file3", + "test/file4" + ); + + getBucket.setClient(ozoneClient); + getBucket.setRequestIdentifier(new RequestIdentifier()); + + ListMultipartUploadsResult result = + (ListMultipartUploadsResult) getBucket.get("b1", "/", null, null, 100, + "test/", null, null, "uploads", null, null, null, 1000).getEntity(); + + // With prefix="test/" and delimiter="/", should have 2 CommonPrefixes and 1 Upload + assertEquals(2, result.getCommonPrefixes().size()); + assertEquals("test/dir1/", result.getCommonPrefixes().get(0).getPrefix().getName()); + assertEquals("test/dir2/", result.getCommonPrefixes().get(1).getPrefix().getName()); + assertEquals(1, result.getUploads().size()); + assertEquals("test/file4", result.getUploads().get(0).getKey().getName()); + } + + @Test + public void listMultipartUploadsWithDelimiterAndPagination() + throws OS3Exception, IOException { + BucketEndpoint getBucket = new BucketEndpoint(); + + OzoneClient ozoneClient = createClientWithMultipartUploads( + "test/dir1/file1", + "test/dir1/file2", + "test/dir1/file3", + "test/dir2/file4", + "test/dir2/file5", + "test/dir2/file6", + "test/dir3/file7", + "test/file8" + ); + + getBucket.setClient(ozoneClient); + getBucket.setRequestIdentifier(new RequestIdentifier()); + + int maxUploads = 2; + + // First page: should get 2 CommonPrefixes (dir1/, dir2/) + ListMultipartUploadsResult result = + (ListMultipartUploadsResult) getBucket.get("b1", "/", null, null, 100, + "test/", null, null, "uploads", null, null, null, maxUploads).getEntity(); + + assertEquals(2, result.getCommonPrefixes().size()); + assertEquals("test/dir1/", result.getCommonPrefixes().get(0).getPrefix().getName()); + assertEquals("test/dir2/", result.getCommonPrefixes().get(1).getPrefix().getName()); + assertEquals(0, result.getUploads().size()); + assertTrue(result.isTruncated()); + assertNotNull(result.getNextKeyMarker()); + + // Second page: should get 1 CommonPrefix (dir3/) and 1 Upload (file8) + result = (ListMultipartUploadsResult) getBucket.get("b1", "/", null, null, 100, + "test/", null, null, "uploads", null, + result.getNextKeyMarker().getName(), result.getNextUploadIdMarker(), maxUploads).getEntity(); + assertEquals(1, result.getCommonPrefixes().size()); + assertEquals("test/dir3/", result.getCommonPrefixes().get(0).getPrefix().getName()); + assertEquals(1, result.getUploads().size()); + assertEquals("test/file8", result.getUploads().get(0).getKey().getName()); + assertFalse(result.isTruncated()); + } + + @Test + public void listMultipartUploadsWithoutDelimiter() throws OS3Exception, IOException { + BucketEndpoint getBucket = new BucketEndpoint(); + + OzoneClient ozoneClient = createClientWithMultipartUploads( + "dir1/file1", + "dir1/file2", + "dir2/file3", + "file4" + ); + + getBucket.setClient(ozoneClient); + getBucket.setRequestIdentifier(new RequestIdentifier()); + + // Without delimiter, all uploads should be returned as individual Upload entries + ListMultipartUploadsResult result = + (ListMultipartUploadsResult) getBucket.get("b1", null, null, null, 100, + "", null, null, "uploads", null, null, null, 1000).getEntity(); + + assertEquals(0, result.getCommonPrefixes().size()); + assertEquals(4, result.getUploads().size()); + } } From be4f355fb3dc30ac82bca819579eb0680f663fbe Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 27 Nov 2025 17:41:41 +0800 Subject: [PATCH 10/10] fix checkstyle --- .../ozone/s3/endpoint/BucketEndpoint.java | 274 +++++++++--------- 1 file changed, 141 insertions(+), 133 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index cd797c1f9447..c7530fe52cdc 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -360,17 +360,8 @@ public Response listMultipartUploads( int maxUploads) throws OS3Exception, IOException { - if (maxUploads < 1) { - throw newError(S3ErrorTable.INVALID_ARGUMENT, "max-uploads", - new Exception("max-uploads must be positive")); - } else { - maxUploads = Math.min(maxUploads, 1000); - } - - // The valid encodingType Values is "url" - if (encodingType != null && !encodingType.equals(ENCODING_TYPE)) { - throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, encodingType); - } + int sanitizedMaxUploads = sanitizeMaxUploads(maxUploads); + validateEncodingType(encodingType); long startNanos = Time.monotonicNowNanos(); S3GAction s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; @@ -380,129 +371,13 @@ public Response listMultipartUploads( try { S3Owner.verifyBucketOwnerCondition(headers, bucketName, bucket.getOwner()); OzoneMultipartUploadList ozoneMultipartUploadList = - bucket.listMultipartUploads(prefix, keyMarker, uploadIdMarker, maxUploads); - - ListMultipartUploadsResult result = new ListMultipartUploadsResult(); - result.setBucket(bucketName); - result.setKeyMarker(EncodingTypeObject.createNullable(keyMarker, encodingType)); - result.setUploadIdMarker(uploadIdMarker); - result.setNextKeyMarker(EncodingTypeObject.createNullable( - ozoneMultipartUploadList.getNextKeyMarker(), encodingType)); - result.setPrefix(EncodingTypeObject.createNullable(prefix, encodingType)); - result.setDelimiter(EncodingTypeObject.createNullable(delimiter, encodingType)); - result.setEncodingType(encodingType); - result.setNextUploadIdMarker(ozoneMultipartUploadList.getNextUploadIdMarker()); - result.setMaxUploads(maxUploads); - result.setTruncated(ozoneMultipartUploadList.isTruncated()); + bucket.listMultipartUploads(prefix, keyMarker, uploadIdMarker, + sanitizedMaxUploads); - final String normalizedPrefix = prefix == null ? "" : prefix; - // Track the previous directory name to avoid duplicate CommonPrefixes entries - String prevDir = null; - // Track the last key processed for pagination - String lastProcessedKey = null; - String lastProcessedUploadId = null; - // Count of items added to response (CommonPrefixes + Uploads) - int responseItemCount = 0; - - List pendingUploads = - ozoneMultipartUploadList.getUploads(); - int processedUploads = 0; - for (OzoneMultipartUpload upload : pendingUploads) { - String keyName = upload.getKeyName(); - - // Filter out keys that don't match the prefix in FS-optimized buckets - if (bucket.getBucketLayout().isFileSystemOptimized() && - StringUtils.isNotEmpty(normalizedPrefix) && - !keyName.startsWith(normalizedPrefix)) { - continue; - } - // Skip keys shorter than prefix to avoid substring errors - if (keyName.length() < normalizedPrefix.length()) { - continue; - } - - // Relative key name after removing the prefix - String relativeKeyName = keyName.substring(normalizedPrefix.length()); - String currentDirName = null; - boolean isDirectoryPlaceholder = false; - if (StringUtils.isNotBlank(delimiter)) { - int depth = StringUtils.countMatches(relativeKeyName, delimiter); - if (depth > 0) { - int delimiterIndex = relativeKeyName.indexOf(delimiter); - currentDirName = relativeKeyName.substring(0, delimiterIndex); - } else if (relativeKeyName.endsWith(delimiter)) { - currentDirName = relativeKeyName.substring( - 0, relativeKeyName.length() - delimiter.length()); - isDirectoryPlaceholder = true; - } - } - - // If we've already returned maxUploads items, keep iterating only to drain - // entries that belong to the same prefix (so that the next page won't repeat them). - if (responseItemCount >= maxUploads) { - if (StringUtils.isNotBlank(delimiter) - && currentDirName != null - && currentDirName.equals(prevDir)) { - lastProcessedKey = keyName; - lastProcessedUploadId = upload.getUploadId(); - continue; - } - break; - } - - // Track whether this upload was added as a CommonPrefix - boolean addedAsPrefix = false; - - // When delimiter is provided, group keys by common prefix (AWS S3 behavior) - // Example: with delimiter="/", keys "dir1/file1" and "dir1/file2" - // are collapsed into CommonPrefix "dir1/" instead of individual Upload entries - if (StringUtils.isNotBlank(delimiter)) { - if (currentDirName != null && !currentDirName.equals(prevDir)) { - result.addCommonPrefix(EncodingTypeObject.createNullable( - normalizedPrefix + currentDirName + delimiter, encodingType)); - prevDir = currentDirName; - responseItemCount++; - addedAsPrefix = true; - } else if (isDirectoryPlaceholder) { - result.addCommonPrefix(EncodingTypeObject.createNullable( - normalizedPrefix + relativeKeyName, encodingType)); - responseItemCount++; - addedAsPrefix = true; - } else if (currentDirName != null) { - addedAsPrefix = true; - } - } - - // Add as individual Upload entry if not collapsed into CommonPrefix - if (!addedAsPrefix) { - result.addUpload(new ListMultipartUploadsResult.Upload( - EncodingTypeObject.createNullable(upload.getKeyName(), encodingType), - upload.getUploadId(), - upload.getCreationTime(), - S3StorageType.fromReplicationConfig(upload.getReplicationConfig()) - )); - responseItemCount++; - } - - // Track the last processed key for pagination - lastProcessedKey = keyName; - lastProcessedUploadId = upload.getUploadId(); - processedUploads++; - } - - boolean hasMoreUploads = - processedUploads < pendingUploads.size() - || ozoneMultipartUploadList.isTruncated(); - - // Override NextKeyMarker and NextUploadIdMarker if we stopped early due to maxUploads - if (responseItemCount >= maxUploads && lastProcessedKey != null - && hasMoreUploads) { - result.setNextKeyMarker(EncodingTypeObject.createNullable(lastProcessedKey, encodingType)); - result.setNextUploadIdMarker(lastProcessedUploadId); - result.setTruncated(true); - } else { - result.setTruncated(ozoneMultipartUploadList.isTruncated()); - } + ListMultipartUploadsResult result = + buildMultipartUploadsResult(bucket, prefix, delimiter, encodingType, + keyMarker, uploadIdMarker, sanitizedMaxUploads, + ozoneMultipartUploadList); AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction, getAuditParameters())); @@ -524,6 +399,139 @@ public Response listMultipartUploads( } } + private int sanitizeMaxUploads(int maxUploads) throws OS3Exception { + if (maxUploads < 1) { + throw newError(S3ErrorTable.INVALID_ARGUMENT, "max-uploads", + new Exception("max-uploads must be positive")); + } + return Math.min(maxUploads, 1000); + } + + private void validateEncodingType(String encodingType) throws OS3Exception { + if (encodingType != null && !encodingType.equals(ENCODING_TYPE)) { + throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, encodingType); + } + } + + private ListMultipartUploadsResult buildMultipartUploadsResult( + OzoneBucket bucket, + String prefix, + String delimiter, + String encodingType, + String keyMarker, + String uploadIdMarker, + int maxUploads, + OzoneMultipartUploadList ozoneMultipartUploadList) { + + ListMultipartUploadsResult result = new ListMultipartUploadsResult(); + result.setBucket(bucket.getName()); + result.setKeyMarker(EncodingTypeObject.createNullable(keyMarker, encodingType)); + result.setUploadIdMarker(uploadIdMarker); + result.setNextKeyMarker(EncodingTypeObject.createNullable( + ozoneMultipartUploadList.getNextKeyMarker(), encodingType)); + result.setPrefix(EncodingTypeObject.createNullable(prefix, encodingType)); + result.setDelimiter(EncodingTypeObject.createNullable(delimiter, encodingType)); + result.setEncodingType(encodingType); + result.setNextUploadIdMarker(ozoneMultipartUploadList.getNextUploadIdMarker()); + result.setMaxUploads(maxUploads); + result.setTruncated(ozoneMultipartUploadList.isTruncated()); + + final String normalizedPrefix = prefix == null ? "" : prefix; + String prevDir = null; + String lastProcessedKey = null; + String lastProcessedUploadId = null; + int responseItemCount = 0; + + List pendingUploads = + ozoneMultipartUploadList.getUploads(); + int processedUploads = 0; + for (OzoneMultipartUpload upload : pendingUploads) { + String keyName = upload.getKeyName(); + + if (bucket.getBucketLayout().isFileSystemOptimized() + && StringUtils.isNotEmpty(normalizedPrefix) + && !keyName.startsWith(normalizedPrefix)) { + continue; + } + if (keyName.length() < normalizedPrefix.length()) { + continue; + } + + String relativeKeyName = keyName.substring(normalizedPrefix.length()); + String currentDirName = null; + boolean isDirectoryPlaceholder = false; + if (StringUtils.isNotBlank(delimiter)) { + int depth = StringUtils.countMatches(relativeKeyName, delimiter); + if (depth > 0) { + int delimiterIndex = relativeKeyName.indexOf(delimiter); + currentDirName = relativeKeyName.substring(0, delimiterIndex); + } else if (relativeKeyName.endsWith(delimiter)) { + currentDirName = relativeKeyName.substring( + 0, relativeKeyName.length() - delimiter.length()); + isDirectoryPlaceholder = true; + } + } + + if (responseItemCount >= maxUploads) { + if (StringUtils.isNotBlank(delimiter) + && currentDirName != null + && currentDirName.equals(prevDir)) { + lastProcessedKey = keyName; + lastProcessedUploadId = upload.getUploadId(); + continue; + } + break; + } + + boolean addedAsPrefix = false; + + if (StringUtils.isNotBlank(delimiter)) { + if (currentDirName != null && !currentDirName.equals(prevDir)) { + result.addCommonPrefix(EncodingTypeObject.createNullable( + normalizedPrefix + currentDirName + delimiter, encodingType)); + prevDir = currentDirName; + responseItemCount++; + addedAsPrefix = true; + } else if (isDirectoryPlaceholder) { + result.addCommonPrefix(EncodingTypeObject.createNullable( + normalizedPrefix + relativeKeyName, encodingType)); + responseItemCount++; + addedAsPrefix = true; + } else if (currentDirName != null) { + addedAsPrefix = true; + } + } + + if (!addedAsPrefix) { + result.addUpload(new ListMultipartUploadsResult.Upload( + EncodingTypeObject.createNullable(upload.getKeyName(), encodingType), + upload.getUploadId(), + upload.getCreationTime(), + S3StorageType.fromReplicationConfig(upload.getReplicationConfig()) + )); + responseItemCount++; + } + + lastProcessedKey = keyName; + lastProcessedUploadId = upload.getUploadId(); + processedUploads++; + } + + boolean hasMoreUploads = + processedUploads < pendingUploads.size() + || ozoneMultipartUploadList.isTruncated(); + + if (responseItemCount >= maxUploads && lastProcessedKey != null + && hasMoreUploads) { + result.setNextKeyMarker(EncodingTypeObject.createNullable(lastProcessedKey, encodingType)); + result.setNextUploadIdMarker(lastProcessedUploadId); + result.setTruncated(true); + } else { + result.setTruncated(ozoneMultipartUploadList.isTruncated()); + } + return result; + } + /** * Rest endpoint to check the existence of a bucket. *