HDDS-13706. Limit max-uploads at 1000 (apache#9060)

Author: adoroszlai

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java

@@ -97,6 +97,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import javax.xml.bind.DatatypeConverter;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.RandomStringUtils;
@@ -128,6 +129,8 @@
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.TestMethodOrder;
 import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 
 /**
  * This is an abstract class to test the AWS Java S3 SDK operations.
@@ -141,6 +144,9 @@
 @TestMethodOrder(MethodOrderer.MethodName.class)
 public abstract class AbstractS3SDKV1Tests extends OzoneTestBase {
 
+  // server-side limitation
+  private static final int MAX_UPLOADS_LIMIT = 1000;
+
   /**
    * There are still some unsupported S3 operations.
    * Current unsupported S3 operations (non-exhaustive):
@@ -758,6 +764,7 @@ public void testListMultipartUploads() {
     uploadIds.add(uploadId3);
 
     ListMultipartUploadsRequest listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName);
+    listMultipartUploadsRequest.setMaxUploads(5000);
 
     MultipartUploadListing result = s3Client.listMultipartUploads(listMultipartUploadsRequest);
 
@@ -768,26 +775,31 @@ public void testListMultipartUploads() {
     assertEquals(uploadIds, listUploadIds);
   }
 
-  @Test
-  public void testListMultipartUploadsPagination() {
-    final String bucketName = getBucketName();
+  @ParameterizedTest
+  @ValueSource(ints = {10, 5000})
+  public void testListMultipartUploadsPagination(int requestedMaxUploads) {
+    final String bucketName = getBucketName() + "-" + requestedMaxUploads;
     final String multipartKeyPrefix = getKeyName("multipart");
 
     s3Client.createBucket(bucketName);
 
-    // Create 25 multipart uploads to test pagination
+    // Create multipart uploads to test pagination
     List<String> allKeys = new ArrayList<>();
     Map<String, String> keyToUploadId = new HashMap<>();
 
-    for (int i = 0; i < 25; i++) {
-      String key = String.format("%s-%03d", multipartKeyPrefix, i);
+    final int effectiveMaxUploads = Math.min(requestedMaxUploads, MAX_UPLOADS_LIMIT);
+    final int uploadsCreated = 2 * effectiveMaxUploads + 5;
+    final int expectedPages = uploadsCreated / effectiveMaxUploads + 1;
+
+    for (int i = 0; i < uploadsCreated; i++) {
+      String key = String.format("%s-%04d", multipartKeyPrefix, i);
       allKeys.add(key);
       String uploadId = initiateMultipartUpload(bucketName, key, null, null, null);
       keyToUploadId.put(key, uploadId);
     }
     Collections.sort(allKeys);
 
-    // Test pagination with maxUploads=10
+    // Test pagination
     Set<String> retrievedKeys = new HashSet<>();
     String keyMarker = null;
     String uploadIdMarker = null;
@@ -796,18 +808,19 @@ public void testListMultipartUploadsPagination() {
 
     do {
       ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(bucketName)
-          .withMaxUploads(10)
+          .withMaxUploads(requestedMaxUploads)
           .withKeyMarker(keyMarker)
           .withUploadIdMarker(uploadIdMarker);
 
       MultipartUploadListing result = s3Client.listMultipartUploads(request);
+      pageCount++;
 
       // Verify page size
-      if (pageCount < 2) {
-        assertEquals(10, result.getMultipartUploads().size());
+      if (pageCount < expectedPages) {
+        assertEquals(effectiveMaxUploads, result.getMultipartUploads().size());
         assertTrue(result.isTruncated());
       } else {
-        assertEquals(5, result.getMultipartUploads().size());
+        assertEquals(uploadsCreated % effectiveMaxUploads, result.getMultipartUploads().size());
         assertFalse(result.isTruncated());
       }
 
@@ -822,7 +835,7 @@ public void testListMultipartUploadsPagination() {
       assertNull(result.getPrefix());
       assertEquals(result.getUploadIdMarker(), uploadIdMarker);
       assertEquals(result.getKeyMarker(), keyMarker);
-      assertEquals(result.getMaxUploads(), 10);
+      assertEquals(effectiveMaxUploads, result.getMaxUploads());
 
       // Verify next markers content
       if (result.isTruncated()) {
@@ -840,32 +853,29 @@ public void testListMultipartUploadsPagination() {
       uploadIdMarker = result.getNextUploadIdMarker();
 
       truncated = result.isTruncated();
-      pageCount++;
 
     } while (truncated);
 
     // Verify pagination results
-    assertEquals(3, pageCount, "Should have exactly 3 pages");
-    assertEquals(25, retrievedKeys.size(), "Should retrieve all uploads");
+    assertEquals(expectedPages, pageCount);
+    assertEquals(uploadsCreated, retrievedKeys.size(), "Should retrieve all uploads");
     assertEquals(
         allKeys,
         retrievedKeys.stream().sorted().collect(Collectors.toList()),
         "Retrieved keys should match expected keys in order");
 
     // Test with prefix
-    String prefix = multipartKeyPrefix + "-01";
+    String prefix = multipartKeyPrefix + "-001";
     ListMultipartUploadsRequest prefixRequest = new ListMultipartUploadsRequest(bucketName)
         .withPrefix(prefix);
 
     MultipartUploadListing prefixResult = s3Client.listMultipartUploads(prefixRequest);
 
     assertEquals(prefix, prefixResult.getPrefix());
     assertEquals(
-        Arrays.asList(multipartKeyPrefix + "-010", multipartKeyPrefix + "-011",
-            multipartKeyPrefix + "-012", multipartKeyPrefix + "-013",
-            multipartKeyPrefix + "-014", multipartKeyPrefix + "-015",
-            multipartKeyPrefix + "-016", multipartKeyPrefix + "-017",
-            multipartKeyPrefix + "-018", multipartKeyPrefix + "-019"),
+        IntStream.rangeClosed(0, 9)
+            .mapToObj(i -> prefix + i)
+            .collect(Collectors.toList()),
         prefixResult.getMultipartUploads().stream()
             .map(MultipartUpload::getKey)
             .collect(Collectors.toList()));

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java

@@ -1228,6 +1228,7 @@ public void testListMultipartUploads() {
         ListMultipartUploadsRequest correctRequest = ListMultipartUploadsRequest.builder()
             .bucket(DEFAULT_BUCKET_NAME)
             .expectedBucketOwner(correctOwner)
+            .maxUploads(5000)
             .build();
         verifyPassBucketOwnershipVerification(() -> s3Client.listMultipartUploads(correctRequest));
 

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java

@@ -355,9 +355,11 @@ public Response listMultipartUploads(
       int maxUploads)
       throws OS3Exception, IOException {
 
-    if (maxUploads < 1 || maxUploads > 1000) {
+    if (maxUploads < 1) {
       throw newError(S3ErrorTable.INVALID_ARGUMENT, "max-uploads",
-          new Exception("max-uploads must be between 1 and 1000"));
+          new Exception("max-uploads must be positive"));
+    } else {
+      maxUploads = Math.min(maxUploads, 1000);
     }
 
     long startNanos = Time.monotonicNowNanos();