Skip to content

Commit 6d4d5eb

Browse files
YutaLinYutaLin
committed
test: snapshot diff with s3 multipart upload modify
1 parent 3a065bf commit 6d4d5eb

File tree

1 file changed

+184
-20
lines changed
  • hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot

1 file changed

+184
-20
lines changed

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java‎

Lines changed: 184 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ public abstract class TestOmSnapshot {
181181
Pattern.compile(SNAPSHOT_KEY_PATTERN_STRING);
182182
private static final int POLL_INTERVAL_MILLIS = 500;
183183
private static final int POLL_MAX_WAIT_MILLIS = 120_000;
184+
private static final int MIN_PART_SIZE = 5 * 1024 * 1024;
184185

185186
private MiniOzoneCluster cluster;
186187
private OzoneClient client;
@@ -3441,8 +3442,6 @@ public void testSnapshotDiffWithCompleteInvisibleMPULifecycle() throws Exception
34413442
assertThrows(OMException.class, () -> bucket.getKey(mpuKey3));
34423443
}
34433444

3444-
private static final int MIN_PART_SIZE = 5 * 1024 * 1024;
3445-
34463445
private void completeSinglePartMPU(OzoneBucket bucket, String keyName, String data) throws IOException {
34473446
OmMultipartInfo mpuInfo = bucket.initiateMultipartUpload(keyName, getDefaultReplication());
34483447
String uploadId = mpuInfo.getUploadID();
@@ -3532,6 +3531,25 @@ private void completeMPUWithReplication(OzoneBucket bucket, String keyName, Repl
35323531
bucket.completeMultipartUpload(keyName, uploadId, partsMap);
35333532
}
35343533

3534+
private void completeMPUWithMetadata(OzoneBucket bucket, String keyName,
3535+
Map<String, String> metadata, Map<String, String> tags) throws IOException {
3536+
3537+
OmMultipartInfo mpuInfo = bucket.initiateMultipartUpload(
3538+
keyName, getDefaultReplication(), metadata, tags);
3539+
String uploadId = mpuInfo.getUploadID();
3540+
3541+
byte[] partData = createLargePartData("MPU with metadata and tags", MIN_PART_SIZE);
3542+
OzoneOutputStream partStream = bucket.createMultipartKey(keyName, partData.length, 1, uploadId);
3543+
partStream.write(partData);
3544+
partStream.close();
3545+
3546+
OzoneMultipartUploadPartListParts partsList = bucket.listParts(keyName, uploadId, 0, 100);
3547+
String realETag = partsList.getPartInfoList().get(0).getPartName();
3548+
3549+
Map<Integer, String> partsMap = Collections.singletonMap(1, realETag);
3550+
bucket.completeMultipartUpload(keyName, uploadId, partsMap);
3551+
}
3552+
35353553
private byte[] createLargePartData(String baseContent, int targetSize) {
35363554
StringBuilder sb = new StringBuilder();
35373555
sb.append(baseContent);
@@ -3540,7 +3558,6 @@ private byte[] createLargePartData(String baseContent, int targetSize) {
35403558
while (sb.length() < targetSize) {
35413559
sb.append(padding);
35423560
if (sb.length() + padding.length() > targetSize) {
3543-
// Add partial padding to reach exact size
35443561
int remaining = targetSize - sb.length();
35453562
sb.append(padding.substring(0, Math.min(remaining, padding.length())));
35463563
}
@@ -3555,7 +3572,7 @@ private byte[] createLargePartData(String baseContent, int targetSize) {
35553572
}
35563573

35573574
@Test
3558-
public void testSnapshotDiffMPU_CreateNewKey() throws Exception {
3575+
public void testSnapshotDiffMPUCreateNewKey() throws Exception {
35593576
String testVolumeName = "vol-create-new-" + counter.incrementAndGet();
35603577
String testBucketName = "bucket-create-new-" + counter.incrementAndGet();
35613578

@@ -3599,7 +3616,7 @@ public void testSnapshotDiffMPU_CreateNewKey() throws Exception {
35993616
}
36003617

36013618
@Test
3602-
public void testSnapshotDiffMPU_CreateMultipleKeys() throws Exception {
3619+
public void testSnapshotDiffMPUCreateMultipleKeys() throws Exception {
36033620
String testVolumeName = "vol-create-multiple-" + counter.incrementAndGet();
36043621
String testBucketName = "bucket-create-multiple-" + counter.incrementAndGet();
36053622

@@ -3664,21 +3681,7 @@ public void testSnapshotDiffMPU_CreateWithMetadataAndTags() throws Exception {
36643681
richTags.put("cost-center", "engineering");
36653682
richTags.put("backup-policy", "daily");
36663683

3667-
OmMultipartInfo mpuInfo = bucket.initiateMultipartUpload(
3668-
mpuKeyWithMeta, getDefaultReplication(), richMetadata, richTags);
3669-
String uploadId = mpuInfo.getUploadID();
3670-
3671-
byte[] partData = createLargePartData("MPU with rich metadata and tags", 1024);
3672-
OzoneDataStreamOutput partStream = bucket.
3673-
createMultipartStreamKey(mpuKeyWithMeta, partData.length, 1, uploadId);
3674-
partStream.write(partData);
3675-
partStream.close();
3676-
3677-
OzoneMultipartUploadPartListParts partsList = bucket.
3678-
listParts(mpuKeyWithMeta, uploadId, 0, 100);
3679-
String realETag = partsList.getPartInfoList().get(0).getPartName();
3680-
Map<Integer, String> partsMap = Collections.singletonMap(1, realETag);
3681-
bucket.completeMultipartUpload(mpuKeyWithMeta, uploadId, partsMap);
3684+
completeMPUWithMetadata(bucket, mpuKeyWithMeta, richMetadata, richTags);
36823685

36833686
String snap2 = "snap-meta-final-" + counter.incrementAndGet();
36843687
createSnapshot(testVolumeName, testBucketName, snap2);
@@ -3722,4 +3725,165 @@ public void testSnapshotDiffMPU_CreateWithDifferentReplication() throws Exceptio
37223725
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, defaultKey));
37233726
assertEquals(expectedDiffs, diff.getDiffList());
37243727
}
3728+
3729+
@Test
3730+
public void testSnapshotDiffMPUModifyExistingKey() throws Exception {
3731+
String testVolumeName = "vol-modify-existing-" + counter.incrementAndGet();
3732+
String testBucketName = "bucket-modify-existing-" + counter.incrementAndGet();
3733+
3734+
store.createVolume(testVolumeName);
3735+
OzoneVolume volume = store.getVolume(testVolumeName);
3736+
createBucket(volume, testBucketName);
3737+
OzoneBucket bucket = volume.getBucket(testBucketName);
3738+
3739+
String existingKey = "existing-key-" + counter.incrementAndGet();
3740+
createFileKey(bucket, existingKey);
3741+
3742+
String snap1 = "snap-before-modify-" + counter.incrementAndGet();
3743+
createSnapshot(testVolumeName, testBucketName, snap1);
3744+
3745+
completeSinglePartMPU(bucket, existingKey, "MPU overwritten content");
3746+
3747+
String snap2 = "snap-after-modify-" + counter.incrementAndGet();
3748+
createSnapshot(testVolumeName, testBucketName, snap2);
3749+
3750+
SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
3751+
3752+
List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Collections.singletonList(
3753+
SnapshotDiffReportOzone.getDiffReportEntry(
3754+
SnapshotDiffReport.DiffType.MODIFY, existingKey));
3755+
assertEquals(expectedDiffs, diff.getDiffList());
3756+
}
3757+
3758+
@Test
3759+
public void testSnapshotDiffMPUModifyMultipleKeys() throws Exception {
3760+
String testVolumeName = "vol-modify-multiple-" + counter.incrementAndGet();
3761+
String testBucketName = "bucket-modify-multiple-" + counter.incrementAndGet();
3762+
3763+
store.createVolume(testVolumeName);
3764+
OzoneVolume volume = store.getVolume(testVolumeName);
3765+
createBucket(volume, testBucketName);
3766+
OzoneBucket bucket = volume.getBucket(testBucketName);
3767+
3768+
String existingKey1 = "existing-1-" + counter.incrementAndGet();
3769+
String existingKey2 = "existing-2-" + counter.incrementAndGet();
3770+
String existingKey3 = "existing-3-" + counter.incrementAndGet();
3771+
String unchangedKey = "unchanged-" + counter.incrementAndGet();
3772+
3773+
createFileKey(bucket, existingKey1);
3774+
createFileKey(bucket, existingKey2);
3775+
createFileKey(bucket, existingKey3);
3776+
createFileKey(bucket, unchangedKey);
3777+
3778+
String snap1 = "snap-modify-multiple-baseline-" + counter.incrementAndGet();
3779+
createSnapshot(testVolumeName, testBucketName, snap1);
3780+
3781+
completeSinglePartMPU(bucket, existingKey1, "Single part overwrite");
3782+
completeMultiplePartMPU(bucket, existingKey2,
3783+
Arrays.asList("Multi part overwrite 1", "Multi part overwrite 2"));
3784+
completeMixedPartMPU(bucket, existingKey3,
3785+
"Mixed regular overwrite", "Mixed stream overwrite");
3786+
3787+
String snap2 = "snap-modify-multiple-final-" + counter.incrementAndGet();
3788+
createSnapshot(testVolumeName, testBucketName, snap2);
3789+
3790+
SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
3791+
3792+
List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
3793+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.MODIFY, existingKey1),
3794+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.MODIFY, existingKey2),
3795+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.MODIFY, existingKey3));
3796+
assertEquals(expectedDiffs, diff.getDiffList());
3797+
}
3798+
3799+
@Test
3800+
public void testSnapshotDiffMPUModifyWithMetadataChange() throws Exception {
3801+
String testVolumeName = "vol-modify-meta-" + counter.incrementAndGet();
3802+
String testBucketName = "bucket-modify-meta-" + counter.incrementAndGet();
3803+
3804+
store.createVolume(testVolumeName);
3805+
OzoneVolume volume = store.getVolume(testVolumeName);
3806+
createBucket(volume, testBucketName);
3807+
OzoneBucket bucket = volume.getBucket(testBucketName);
3808+
3809+
String existingKey = "existing-meta-key-" + counter.incrementAndGet();
3810+
createFileKey(bucket, existingKey);
3811+
3812+
Map<String, String> originalTags = new HashMap<>();
3813+
originalTags.put("version", "1.0");
3814+
originalTags.put("environment", "test");
3815+
bucket.putObjectTagging(existingKey, originalTags);
3816+
3817+
String snap1 = "snap-modify-meta-baseline-" + counter.incrementAndGet();
3818+
createSnapshot(testVolumeName, testBucketName, snap1);
3819+
3820+
Map<String, String> newMetadata = new HashMap<>();
3821+
newMetadata.put("content-type", "application/json");
3822+
newMetadata.put("updated-by", "mpu-test");
3823+
3824+
Map<String, String> newTags = new HashMap<>();
3825+
newTags.put("version", "2.0");
3826+
newTags.put("environment", "updated");
3827+
newTags.put("method", "mpu-overwrite");
3828+
3829+
completeMPUWithMetadata(bucket, existingKey, newMetadata, newTags);
3830+
3831+
String snap2 = "snap-modify-meta-final-" + counter.incrementAndGet();
3832+
createSnapshot(testVolumeName, testBucketName, snap2);
3833+
3834+
SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
3835+
3836+
List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
3837+
SnapshotDiffReportOzone.getDiffReportEntry(
3838+
SnapshotDiffReport.DiffType.MODIFY, existingKey));
3839+
assertEquals(expectedDiffs, diff.getDiffList());
3840+
}
3841+
3842+
@Test
3843+
public void testSnapshotDiffMPUMixedCreateAndModify() throws Exception {
3844+
String testVolumeName = "vol-mixed-ops-" + counter.incrementAndGet();
3845+
String testBucketName = "bucket-mixed-ops-" + counter.incrementAndGet();
3846+
3847+
store.createVolume(testVolumeName);
3848+
OzoneVolume volume = store.getVolume(testVolumeName);
3849+
createBucket(volume, testBucketName);
3850+
OzoneBucket bucket = volume.getBucket(testBucketName);
3851+
3852+
String existingKey1 = "existing-1-" + counter.incrementAndGet();
3853+
String existingKey2 = "existing-2-" + counter.incrementAndGet();
3854+
String unchangedKey = "unchanged-" + counter.incrementAndGet();
3855+
3856+
createFileKey(bucket, existingKey1);
3857+
createFileKey(bucket, existingKey2);
3858+
createFileKey(bucket, unchangedKey);
3859+
3860+
String snap1 = "snap-mixed-baseline-" + counter.incrementAndGet();
3861+
createSnapshot(testVolumeName, testBucketName, snap1);
3862+
3863+
String newKey1 = "new-mpu-1-" + counter.incrementAndGet();
3864+
String newKey2 = "new-mpu-2-" + counter.incrementAndGet();
3865+
3866+
completeSinglePartMPU(bucket, newKey1, "New key via single part MPU");
3867+
completeMixedPartMPU(bucket, newKey2, "New key regular part", "New key stream part");
3868+
3869+
completeSinglePartMPU(bucket, existingKey1, "Modified via single part MPU");
3870+
completeMultiplePartMPU(bucket, existingKey2,
3871+
Arrays.asList("Modified part 1", "Modified part 2"));
3872+
3873+
String regularNewKey = "regular-new-" + counter.incrementAndGet();
3874+
createFileKey(bucket, regularNewKey);
3875+
3876+
String snap2 = "snap-mixed-final-" + counter.incrementAndGet();
3877+
createSnapshot(testVolumeName, testBucketName, snap2);
3878+
3879+
SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
3880+
3881+
List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
3882+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, newKey1),
3883+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, newKey2),
3884+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, regularNewKey),
3885+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.MODIFY, existingKey1),
3886+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.MODIFY, existingKey2));
3887+
assertEquals(expectedDiffs, diff.getDiffList());
3888+
}
37253889
}

0 commit comments

Comments
 (0)