Skip to content

Commit a5d863a

Browse files
YutaLinYutaLin
committed
test: snapshot diff with mupltipart upload
1 parent e9492e2 commit a5d863a

File tree

1 file changed

+283
-0
lines changed
  • hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot

1 file changed

+283
-0
lines changed

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java‎

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@
139139
import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
140140
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
141141
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
142+
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
142143
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
143144
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
144145
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
@@ -3439,4 +3440,286 @@ public void testSnapshotDiffWithCompleteInvisibleMPULifecycle() throws Exception
34393440
assertThrows(OMException.class, () -> bucket.getKey(mpuKey2));
34403441
assertThrows(OMException.class, () -> bucket.getKey(mpuKey3));
34413442
}
3443+
3444+
private static final int MIN_PART_SIZE = 5 * 1024 * 1024;
3445+
3446+
private void completeSinglePartMPU(OzoneBucket bucket, String keyName, String data) throws IOException {
3447+
OmMultipartInfo mpuInfo = bucket.initiateMultipartUpload(keyName, getDefaultReplication());
3448+
String uploadId = mpuInfo.getUploadID();
3449+
3450+
byte[] partData = createLargePartData(data, MIN_PART_SIZE);
3451+
OzoneOutputStream partStream = bucket.createMultipartKey(keyName, partData.length, 1, uploadId);
3452+
partStream.write(partData);
3453+
partStream.close();
3454+
3455+
OzoneMultipartUploadPartListParts partsList = bucket.listParts(keyName, uploadId, 0, 100);
3456+
String realETag = partsList.getPartInfoList().get(0).getPartName();
3457+
3458+
Map<Integer, String> partsMap = Collections.singletonMap(1, realETag);
3459+
bucket.completeMultipartUpload(keyName, uploadId, partsMap);
3460+
}
3461+
3462+
private void completeMultiplePartMPU(OzoneBucket bucket, String keyName, List<String> partDataList) throws IOException {
3463+
OmMultipartInfo mpuInfo = bucket.initiateMultipartUpload(keyName, getDefaultReplication());
3464+
String uploadId = mpuInfo.getUploadID();
3465+
3466+
for (int i = 0; i < partDataList.size(); i++) {
3467+
int partNum = i + 1;
3468+
byte[] partData = createLargePartData(partDataList.get(i), MIN_PART_SIZE);
3469+
3470+
try (OzoneOutputStream partStream = bucket.createMultipartKey(
3471+
keyName, partData.length, partNum, uploadId)) {
3472+
partStream.write(partData);
3473+
}
3474+
}
3475+
3476+
OzoneMultipartUploadPartListParts partsList = bucket.listParts(keyName, uploadId, 0, partDataList.size());
3477+
Map<Integer, String> partsMap = new HashMap<>();
3478+
3479+
for (OzoneMultipartUploadPartListParts.PartInfo partInfo : partsList.getPartInfoList()) {
3480+
partsMap.put(partInfo.getPartNumber(), partInfo.getPartName());
3481+
}
3482+
3483+
bucket.completeMultipartUpload(keyName, uploadId, partsMap);
3484+
}
3485+
3486+
private void completeMixedPartMPU(OzoneBucket bucket, String keyName, String regularData, String streamData) throws IOException {
3487+
OmMultipartInfo mpuInfo = bucket.initiateMultipartUpload(keyName, getDefaultReplication());
3488+
String uploadId = mpuInfo.getUploadID();
3489+
3490+
byte[] part1Data = createLargePartData(regularData, MIN_PART_SIZE);
3491+
try (OzoneOutputStream partStream = bucket.createMultipartKey(
3492+
keyName, part1Data.length, 1, uploadId)) {
3493+
partStream.write(part1Data);
3494+
}
3495+
3496+
byte[] part2Data = createLargePartData(streamData, MIN_PART_SIZE);
3497+
try (OzoneDataStreamOutput partStream = bucket.createMultipartStreamKey(
3498+
keyName, part2Data.length, 2, uploadId)) {
3499+
partStream.write(part2Data);
3500+
}
3501+
3502+
OzoneMultipartUploadPartListParts partsList = bucket.listParts(keyName, uploadId, 0, 2);
3503+
Map<Integer, String> partsMap = new HashMap<>();
3504+
3505+
for (OzoneMultipartUploadPartListParts.PartInfo partInfo : partsList.getPartInfoList()) {
3506+
partsMap.put(partInfo.getPartNumber(), partInfo.getPartName());
3507+
}
3508+
3509+
bucket.completeMultipartUpload(keyName, uploadId, partsMap);
3510+
}
3511+
3512+
private void completeMPUWithReplication(OzoneBucket bucket, String keyName, ReplicationConfig replicationConfig) throws IOException {
3513+
OmMultipartInfo mpuInfo;
3514+
if (replicationConfig != null) {
3515+
mpuInfo = bucket.initiateMultipartUpload(keyName, replicationConfig);
3516+
} else {
3517+
mpuInfo = bucket.initiateMultipartUpload(keyName);
3518+
}
3519+
3520+
String uploadId = mpuInfo.getUploadID();
3521+
3522+
byte[] partData = createLargePartData("Replication test data for " + keyName, MIN_PART_SIZE);
3523+
try (OzoneOutputStream partStream = bucket.createMultipartKey(
3524+
keyName, partData.length, 1, uploadId)) {
3525+
partStream.write(partData);
3526+
}
3527+
3528+
OzoneMultipartUploadPartListParts partsList = bucket.listParts(keyName, uploadId, 0, 1);
3529+
String realETag = partsList.getPartInfoList().get(0).getPartName();
3530+
3531+
Map<Integer, String> partsMap = Collections.singletonMap(1, realETag);
3532+
bucket.completeMultipartUpload(keyName, uploadId, partsMap);
3533+
}
3534+
3535+
private byte[] createLargePartData(String baseContent, int targetSize) {
3536+
StringBuilder sb = new StringBuilder();
3537+
sb.append(baseContent);
3538+
3539+
String padding = " - padding data";
3540+
while (sb.length() < targetSize) {
3541+
sb.append(padding);
3542+
if (sb.length() + padding.length() > targetSize) {
3543+
// Add partial padding to reach exact size
3544+
int remaining = targetSize - sb.length();
3545+
sb.append(padding.substring(0, Math.min(remaining, padding.length())));
3546+
}
3547+
}
3548+
3549+
String result = sb.toString();
3550+
if (result.length() > targetSize) {
3551+
result = result.substring(0, targetSize);
3552+
}
3553+
3554+
return result.getBytes(UTF_8);
3555+
}
3556+
3557+
@Test
3558+
public void testSnapshotDiffMPU_CreateNewKey() throws Exception {
3559+
String testVolumeName = "vol-create-new-" + counter.incrementAndGet();
3560+
String testBucketName = "bucket-create-new-" + counter.incrementAndGet();
3561+
3562+
store.createVolume(testVolumeName);
3563+
OzoneVolume volume = store.getVolume(testVolumeName);
3564+
createBucket(volume, testBucketName);
3565+
OzoneBucket bucket = volume.getBucket(testBucketName);
3566+
3567+
String existingKey = "existing-baseline-" + counter.incrementAndGet();
3568+
createFileKey(bucket, existingKey);
3569+
3570+
String snap1 = "snap-before-create-" + counter.incrementAndGet();
3571+
createSnapshot(testVolumeName, testBucketName, snap1);
3572+
3573+
String mpuKey1 = "multi-mpu-1-" + counter.incrementAndGet();
3574+
String mpuKey2 = "multi-mpu-2-" + counter.incrementAndGet();
3575+
String mpuKey3 = "multi-mpu-3-" + counter.incrementAndGet();
3576+
3577+
completeSinglePartMPU(bucket, mpuKey1, "Single part MPU data");
3578+
3579+
completeMultiplePartMPU(bucket, mpuKey2,
3580+
Arrays.asList("Multi part 1", "Multi part 2", "Multi part 3"));
3581+
3582+
completeMixedPartMPU(bucket, mpuKey3,
3583+
"Mixed regular part", "Mixed stream part");
3584+
3585+
String regularKey = "regular-compare-" + counter.incrementAndGet();
3586+
createFileKey(bucket, regularKey);
3587+
3588+
String snap2 = "snap-multiple-final-" + counter.incrementAndGet();
3589+
createSnapshot(testVolumeName, testBucketName, snap2);
3590+
3591+
SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
3592+
3593+
List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
3594+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, mpuKey1),
3595+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, mpuKey2),
3596+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, mpuKey3),
3597+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, regularKey));
3598+
assertEquals(expectedDiffs, diff.getDiffList());
3599+
}
3600+
3601+
@Test
3602+
public void testSnapshotDiffMPU_CreateMultipleKeys() throws Exception {
3603+
String testVolumeName = "vol-create-multiple-" + counter.incrementAndGet();
3604+
String testBucketName = "bucket-create-multiple-" + counter.incrementAndGet();
3605+
3606+
store.createVolume(testVolumeName);
3607+
OzoneVolume volume = store.getVolume(testVolumeName);
3608+
createBucket(volume, testBucketName);
3609+
OzoneBucket bucket = volume.getBucket(testBucketName);
3610+
3611+
String snap1 = "snap-multiple-baseline-" + counter.incrementAndGet();
3612+
createSnapshot(testVolumeName, testBucketName, snap1);
3613+
3614+
String mpuKey1 = "multi-mpu-1-" + counter.incrementAndGet();
3615+
String mpuKey2 = "multi-mpu-2-" + counter.incrementAndGet();
3616+
String mpuKey3 = "multi-mpu-3-" + counter.incrementAndGet();
3617+
3618+
completeSinglePartMPU(bucket, mpuKey1, "Single part MPU data");
3619+
completeMultiplePartMPU(bucket, mpuKey2,
3620+
Arrays.asList("Multi part 1", "Multi part 2", "Multi part 3"));
3621+
completeMixedPartMPU(bucket, mpuKey3,
3622+
"Mixed regular part", "Mixed stream part");
3623+
3624+
String regularKey = "regular-compare-" + counter.incrementAndGet();
3625+
createFileKey(bucket, regularKey);
3626+
3627+
String snap2 = "snap-multiple-final-" + counter.incrementAndGet();
3628+
createSnapshot(testVolumeName, testBucketName, snap2);
3629+
3630+
SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
3631+
3632+
List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
3633+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, mpuKey1),
3634+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, mpuKey2),
3635+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, mpuKey3),
3636+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, regularKey));
3637+
assertEquals(expectedDiffs, diff.getDiffList());
3638+
}
3639+
3640+
@Test
3641+
public void testSnapshotDiffMPU_CreateWithMetadataAndTags() throws Exception {
3642+
String testVolumeName = "vol-create-meta-" + counter.incrementAndGet();
3643+
String testBucketName = "bucket-create-meta-" + counter.incrementAndGet();
3644+
3645+
store.createVolume(testVolumeName);
3646+
OzoneVolume volume = store.getVolume(testVolumeName);
3647+
createBucket(volume, testBucketName);
3648+
OzoneBucket bucket = volume.getBucket(testBucketName);
3649+
3650+
String snap1 = "snap-meta-baseline-" + counter.incrementAndGet();
3651+
createSnapshot(testVolumeName, testBucketName, snap1);
3652+
3653+
String mpuKeyWithMeta = "mpu-with-metadata-" + counter.incrementAndGet();
3654+
3655+
Map<String, String> richMetadata = new HashMap<>();
3656+
richMetadata.put("content-type", "application/octet-stream");
3657+
richMetadata.put("content-encoding", "gzip");
3658+
richMetadata.put("cache-control", "no-cache");
3659+
richMetadata.put("custom-header", "test-value");
3660+
3661+
Map<String, String> richTags = new HashMap<>();
3662+
richTags.put("project", "ozone-testing");
3663+
richTags.put("environment", "integration");
3664+
richTags.put("cost-center", "engineering");
3665+
richTags.put("backup-policy", "daily");
3666+
3667+
OmMultipartInfo mpuInfo = bucket.initiateMultipartUpload(
3668+
mpuKeyWithMeta, getDefaultReplication(), richMetadata, richTags);
3669+
String uploadId = mpuInfo.getUploadID();
3670+
3671+
byte[] partData = createLargePartData("MPU with rich metadata and tags", 1024);
3672+
OzoneDataStreamOutput partStream = bucket.
3673+
createMultipartStreamKey(mpuKeyWithMeta, partData.length, 1, uploadId);
3674+
partStream.write(partData);
3675+
partStream.close();
3676+
3677+
OzoneMultipartUploadPartListParts partsList = bucket.
3678+
listParts(mpuKeyWithMeta, uploadId, 0, 100);
3679+
String realETag = partsList.getPartInfoList().get(0).getPartName();
3680+
Map<Integer, String> partsMap = Collections.singletonMap(1, realETag);
3681+
bucket.completeMultipartUpload(mpuKeyWithMeta, uploadId, partsMap);
3682+
3683+
String snap2 = "snap-meta-final-" + counter.incrementAndGet();
3684+
createSnapshot(testVolumeName, testBucketName, snap2);
3685+
3686+
SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
3687+
3688+
List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
3689+
SnapshotDiffReportOzone.getDiffReportEntry(
3690+
SnapshotDiffReport.DiffType.CREATE, mpuKeyWithMeta));
3691+
assertEquals(expectedDiffs, diff.getDiffList());
3692+
}
3693+
3694+
@Test
3695+
public void testSnapshotDiffMPU_CreateWithDifferentReplication() throws Exception {
3696+
String testVolumeName = "vol-create-repl-" + counter.incrementAndGet();
3697+
String testBucketName = "bucket-create-repl-" + counter.incrementAndGet();
3698+
3699+
store.createVolume(testVolumeName);
3700+
OzoneVolume volume = store.getVolume(testVolumeName);
3701+
createBucket(volume, testBucketName);
3702+
OzoneBucket bucket = volume.getBucket(testBucketName);
3703+
3704+
String snap1 = "snap-repl-baseline-" + counter.incrementAndGet();
3705+
createSnapshot(testVolumeName, testBucketName, snap1);
3706+
3707+
String standaloneKey = "standalone-mpu-" + counter.incrementAndGet();
3708+
String defaultKey = "default-mpu-" + counter.incrementAndGet();
3709+
3710+
ReplicationConfig standaloneConfig = StandaloneReplicationConfig.getInstance(
3711+
HddsProtos.ReplicationFactor.ONE);
3712+
completeMPUWithReplication(bucket, standaloneKey, standaloneConfig);
3713+
completeMPUWithReplication(bucket, defaultKey, null);
3714+
3715+
String snap2 = "snap-repl-final-" + counter.incrementAndGet();
3716+
createSnapshot(testVolumeName, testBucketName, snap2);
3717+
3718+
SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
3719+
3720+
List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
3721+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, standaloneKey),
3722+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE, defaultKey));
3723+
assertEquals(expectedDiffs, diff.getDiffList());
3724+
}
34423725
}

0 commit comments

Comments
 (0)