Skip to content

Commit 72aefc2

Browse files
YutaLinYutaLin
committed
test: add snapshot diff test with stream file
1 parent e4f780b commit 72aefc2

File tree

1 file changed

+195
-0
lines changed
  • hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot

1 file changed

+195
-0
lines changed

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2969,4 +2969,199 @@ public void testSnapDiffWithMixedStreamAndRegularKeys() throws Exception {
29692969
SnapshotDiffReport.DiffType.CREATE, streamKey2));
29702970
assertEquals(expectedDiffs, diff.getDiffList());
29712971
}
2972+
2973+
private String createStreamFileWithPrefix(OzoneBucket bucket, String filePrefix)
2974+
throws Exception {
2975+
String file = filePrefix + counter.incrementAndGet();
2976+
return createStreamFile(bucket, file, false);
2977+
}
2978+
2979+
private String createStreamFile(OzoneBucket bucket, String fileName, boolean overWrite)
2980+
throws Exception {
2981+
byte[] value = RandomStringUtils.secure().nextAscii(10240).getBytes(UTF_8);
2982+
OzoneDataStreamOutput streamFile = bucket.createStreamFile(
2983+
fileName, value.length, bucket.getReplicationConfig(), overWrite, true);
2984+
streamFile.write(value);
2985+
streamFile.close();
2986+
GenericTestUtils.waitFor(() -> {
2987+
try {
2988+
getOmKeyInfo(bucket.getVolumeName(), bucket.getName(), fileName);
2989+
} catch (IOException e) {
2990+
return false;
2991+
}
2992+
return true;
2993+
}, 300, 10000);
2994+
return fileName;
2995+
}
2996+
2997+
@Test
2998+
public void testSnapDiffWithStreamFileModification() throws Exception {
2999+
assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths));
3000+
3001+
String testVolumeName = "vol" + counter.incrementAndGet();
3002+
String testBucketName = "bucket1";
3003+
store.createVolume(testVolumeName);
3004+
OzoneVolume volume = store.getVolume(testVolumeName);
3005+
createBucket(volume, testBucketName);
3006+
OzoneBucket bucket = volume.getBucket(testBucketName);
3007+
3008+
String file1 = "stream-file-1";
3009+
file1 = createStreamFileWithPrefix(bucket, file1);
3010+
String snap1 = "snap1";
3011+
createSnapshot(testVolumeName, testBucketName, snap1);
3012+
3013+
String file2 = "stream-file-2";
3014+
file2 = createStreamFileWithPrefix(bucket, file2);
3015+
bucket.deleteKey(file1);
3016+
String snap2 = "snap2";
3017+
createSnapshot(testVolumeName, testBucketName, snap2);
3018+
3019+
SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
3020+
testBucketName, snap1, snap2);
3021+
assertEquals(2, diff.getDiffList().size());
3022+
assertEquals(Arrays.asList(
3023+
SnapshotDiffReportOzone.getDiffReportEntry(
3024+
SnapshotDiffReport.DiffType.DELETE, file1),
3025+
SnapshotDiffReportOzone.getDiffReportEntry(
3026+
SnapshotDiffReport.DiffType.CREATE, file2)),
3027+
diff.getDiffList());
3028+
}
3029+
3030+
@Test
3031+
public void testSnapDiffWithStreamFileRewrite() throws Exception {
3032+
assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths));
3033+
3034+
String testVolumeName = "vol" + counter.incrementAndGet();
3035+
String testBucketName = "bucket1";
3036+
store.createVolume(testVolumeName);
3037+
OzoneVolume volume = store.getVolume(testVolumeName);
3038+
createBucket(volume, testBucketName);
3039+
OzoneBucket bucket = volume.getBucket(testBucketName);
3040+
3041+
String file1 = "stream-file-1";
3042+
file1 = createStreamFileWithPrefix(bucket, file1);
3043+
String snap1 = "snap1";
3044+
createSnapshot(testVolumeName, testBucketName, snap1);
3045+
3046+
createStreamFile(bucket, file1, true);
3047+
String snap2 = "snap2";
3048+
createSnapshot(testVolumeName, testBucketName, snap2);
3049+
3050+
SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
3051+
testBucketName, snap1, snap2);
3052+
assertEquals(1, diff.getDiffList().size());
3053+
assertEquals(Collections.singletonList(
3054+
SnapshotDiffReportOzone.getDiffReportEntry(
3055+
SnapshotDiffReport.DiffType.MODIFY, file1)),
3056+
diff.getDiffList());
3057+
}
3058+
3059+
@Test
3060+
public void testSnapDiffWithStreamFileRename() throws Exception {
3061+
assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths));
3062+
3063+
String testVolumeName = "vol" + counter.incrementAndGet();
3064+
String testBucketName = "bucket1";
3065+
store.createVolume(testVolumeName);
3066+
OzoneVolume volume = store.getVolume(testVolumeName);
3067+
createBucket(volume, testBucketName);
3068+
OzoneBucket bucket = volume.getBucket(testBucketName);
3069+
3070+
String file1 = "stream-file-1";
3071+
file1 = createStreamFileWithPrefix(bucket, file1);
3072+
String snap1 = "snap1";
3073+
createSnapshot(testVolumeName, testBucketName, snap1);
3074+
3075+
String renamedFile = file1 + "_renamed";
3076+
bucket.renameKey(file1, renamedFile);
3077+
GenericTestUtils.waitFor(() -> {
3078+
try {
3079+
getOmKeyInfo(testVolumeName, testBucketName, renamedFile);
3080+
} catch (IOException e) {
3081+
return false;
3082+
}
3083+
return true;
3084+
}, 1000, 30000);
3085+
3086+
String snap2 = "snap2";
3087+
createSnapshot(testVolumeName, testBucketName, snap2);
3088+
3089+
SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
3090+
testBucketName, snap1, snap2);
3091+
assertEquals(1, diff.getDiffList().size());
3092+
assertEquals(Collections.singletonList(
3093+
SnapshotDiffReportOzone.getDiffReportEntry(
3094+
SnapshotDiffReport.DiffType.RENAME, file1, renamedFile)),
3095+
diff.getDiffList());
3096+
}
3097+
3098+
@Test
3099+
public void testSnapDiffWithStreamFileRecreation() throws Exception {
3100+
assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths));
3101+
3102+
String testVolumeName = "vol" + counter.incrementAndGet();
3103+
String testBucketName = "bucket1";
3104+
store.createVolume(testVolumeName);
3105+
OzoneVolume volume = store.getVolume(testVolumeName);
3106+
createBucket(volume, testBucketName);
3107+
OzoneBucket bucket = volume.getBucket(testBucketName);
3108+
3109+
String file1 = "stream-file-1";
3110+
file1 = createStreamFileWithPrefix(bucket, file1);
3111+
String snap1 = "snap1";
3112+
createSnapshot(testVolumeName, testBucketName, snap1);
3113+
3114+
getOmKeyInfo(testVolumeName, testBucketName, file1);
3115+
bucket.deleteKey(file1);
3116+
file1 = createStreamFile(bucket, file1, true);
3117+
getOmKeyInfo(testVolumeName, testBucketName, file1);
3118+
String snap2 = "snap2";
3119+
createSnapshot(testVolumeName, testBucketName, snap2);
3120+
3121+
SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
3122+
testBucketName, snap1, snap2);
3123+
assertEquals(2, diff.getDiffList().size());
3124+
assertEquals(Arrays.asList(
3125+
SnapshotDiffReportOzone.getDiffReportEntry(
3126+
SnapshotDiffReport.DiffType.DELETE, file1),
3127+
SnapshotDiffReportOzone.getDiffReportEntry(
3128+
SnapshotDiffReport.DiffType.CREATE, file1)),
3129+
diff.getDiffList());
3130+
}
3131+
3132+
@Test
3133+
public void testSnapDiffWithStreamFileAndDirectoryOperations() throws Exception {
3134+
assumeTrue(bucketLayout.isFileSystemOptimized());
3135+
String testVolumeName = "vol" + counter.incrementAndGet();
3136+
String testBucketName = "bucket1";
3137+
store.createVolume(testVolumeName);
3138+
OzoneVolume volume = store.getVolume(testVolumeName);
3139+
createBucket(volume, testBucketName);
3140+
OzoneBucket bucket = volume.getBucket(testBucketName);
3141+
3142+
bucket.createDirectory("dir1/dir2");
3143+
String file1 = "dir1/dir2/stream-file-1";
3144+
createStreamFile(bucket, file1, false);
3145+
String snap1 = "snap1";
3146+
createSnapshot(testVolumeName, testBucketName, snap1);
3147+
3148+
String file2 = "dir1/dir2/stream-file-2";
3149+
createStreamFile(bucket, file2, false);
3150+
bucket.renameKey("dir1/dir2", "dir1/dir3");
3151+
String snap2 = "snap2";
3152+
createSnapshot(testVolumeName, testBucketName, snap2);
3153+
3154+
SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
3155+
testBucketName, snap1, snap2);
3156+
assertEquals(3, diff.getDiffList().size());
3157+
3158+
List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
3159+
SnapshotDiffReportOzone.getDiffReportEntry(
3160+
SnapshotDiffReport.DiffType.RENAME, "dir1/dir2", "dir1/dir3"),
3161+
SnapshotDiffReportOzone.getDiffReportEntry(
3162+
SnapshotDiffReport.DiffType.CREATE, "dir1/dir3/stream-file-2"),
3163+
SnapshotDiffReportOzone.getDiffReportEntry(
3164+
SnapshotDiffReport.DiffType.MODIFY, "dir1"));
3165+
assertEquals(expectedDiffs, diff.getDiffList());
3166+
}
29723167
}

0 commit comments

Comments
 (0)