From 69f19de1ad143d8388bb225a996f12946f1cd11d Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Tue, 24 May 2022 15:07:47 +0300 Subject: [PATCH 01/19] IGNITE-15067 Change metastorage key. --- .../snapshot/IgniteSnapshotManager.java | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index ae9d339da6f28c..ade30370ee60bf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -259,9 +259,6 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter /** Error message to finalize snapshot tasks. */ public static final String SNP_NODE_STOPPING_ERR_MSG = "The operation is cancelled due to the local node is stopping"; - /** Metastorage key to save currently running snapshot. */ - public static final String SNP_RUNNING_KEY = "snapshot-running"; - /** Snapshot metrics prefix. */ public static final String SNAPSHOT_METRICS = "snapshot"; @@ -280,6 +277,17 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter /** Maximum block size for limited snapshot transfer (64KB by default). */ public static final int SNAPSHOT_LIMITED_TRANSFER_BLOCK_SIZE_BYTES = 64 * 1024; + /** Metastorage key to save currently running snapshot directory. */ + private static final String SNP_RUNNING_DIR_KEY = "snapshot-running-dir"; + + /** + * Metastorage key to save currently running snapshot. + * + * @deprecated Use #SNP_RUNNING_DIR_KEY instead. + */ + @Deprecated + private static final String SNP_RUNNING_KEY = "snapshot-running"; + /** Snapshot operation finish log message. */ private static final String SNAPSHOT_FINISHED_MSG = "Cluster-wide snapshot operation finished successfully: "; @@ -1546,20 +1554,22 @@ public List readSnapshotMetadatas(String snpName) { // Snapshot which has not been completed due to the local node crashed must be deleted. String snpName = (String)metaStorage.read(SNP_RUNNING_KEY); + String snpDirName = snpName == null ? (String)metaStorage.read(SNP_RUNNING_DIR_KEY) : null; - if (snpName == null) - return; + File snpDir = snpName != null ? snapshotLocalDir(snpName) : snpDirName != null ? new File(snpDirName) : null; - recovered = true; + if (snpDir != null) { + recovered = true; - for (File tmp : snapshotTmpDir().listFiles()) - U.delete(tmp); + for (File tmp : snapshotTmpDir().listFiles()) + U.delete(tmp); - deleteSnapshot(snapshotLocalDir(snpName), pdsSettings.folderName()); + deleteSnapshot(snpDir, pdsSettings.folderName()); - if (log.isInfoEnabled()) { - log.info("Previous attempt to create snapshot fail due to the local node crash. All resources " + - "related to snapshot operation have been deleted: " + snpName); + if (log.isInfoEnabled()) { + log.info("Previous attempt to create snapshot fail due to the local node crash. All resources " + + "related to snapshot operation have been deleted: " + snpDir.getName()); + } } } @@ -1891,6 +1901,7 @@ private void removeLastMetaStorageKey() throws IgniteCheckedException { cctx.database().checkpointReadLock(); try { + metaStorage.remove(SNP_RUNNING_DIR_KEY); metaStorage.remove(SNP_RUNNING_KEY); } finally { @@ -3085,10 +3096,10 @@ public LocalSnapshotSender(String snpName) { cctx.database().checkpointReadLock(); try { - assert metaStorage != null && metaStorage.read(SNP_RUNNING_KEY) == null : + assert metaStorage != null && metaStorage.read(SNP_RUNNING_DIR_KEY) == null : "The previous snapshot hasn't been completed correctly"; - metaStorage.write(SNP_RUNNING_KEY, snpName); + metaStorage.write(SNP_RUNNING_DIR_KEY, snpLocDir.toString()); U.ensureDirectory(dbDir, "snapshot work directory for a local snapshot sender", log); } From 93b7ed79de279b43fe462e6c8cd71f2b6e406ed8 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Tue, 24 May 2022 18:59:45 +0300 Subject: [PATCH 02/19] IGNITE-15067 Pass path as parameter (wip). --- .../snapshot/IgniteSnapshotManager.java | 39 ++++++++++++++----- .../snapshot/SnapshotOperationRequest.java | 16 +++++++- .../snapshot/SnapshotRestoreProcess.java | 4 +- .../snapshot/AbstractSnapshotSelfTest.java | 6 +-- .../snapshot/EncryptedSnapshotTest.java | 2 +- .../IgniteClusterSnapshotSelfTest.java | 36 +++++++++++------ .../IgniteSnapshotManagerSelfTest.java | 8 ++-- .../snapshot/PlainSnapshotTest.java | 2 +- 8 files changed, 80 insertions(+), 33 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index ade30370ee60bf..47b12a412ae581 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -354,7 +354,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter private volatile ReadWriteMetastorage metaStorage; /** Local snapshot sender factory. */ - private Function locSndrFactory = LocalSnapshotSender::new; + private BiFunction locSndrFactory = LocalSnapshotSender::new; /** Remote snapshot sender factory. */ private BiFunction rmtSndrFactory = this::remoteSnapshotSenderFactory; @@ -689,6 +689,15 @@ public void deleteSnapshot(File snpDir, String folderName) { } } + public File snapshotLocalDir(SnapshotOperationRequest req) { + assert locSnpDir != null; + assert U.alphanumericUnderscore(req.snapshotName()) : req.snapshotName(); + + File locSnpDir0 = req.snapshotPath(); + + return new File(locSnpDir0 == null ? locSnpDir : locSnpDir0, req.snapshotName()); + } + /** * @param snpName Snapshot name. * @return Local snapshot directory for snapshot with given name. @@ -789,7 +798,7 @@ private IgniteInternalFuture initLocalSnapshotStartSt req.operationalNodeId(), parts, withMetaStorage, - locSndrFactory.apply(req.snapshotName())); + locSndrFactory.apply(req.snapshotName(), req.snapshotPath())); if (withMetaStorage && task0 instanceof SnapshotFutureTask) { ((DistributedMetaStorageImpl)cctx.kernalContext().distributedMetastorage()) @@ -808,7 +817,7 @@ private IgniteInternalFuture initLocalSnapshotStartSt .map(n -> cctx.discovery().node(n).consistentId().toString()) .collect(Collectors.toSet()); - File smf = new File(snapshotLocalDir(req.snapshotName()), snapshotMetaFileName(cctx.localNode().consistentId().toString())); + File smf = new File(snapshotLocalDir(req), snapshotMetaFileName(cctx.localNode().consistentId().toString())); if (smf.exists()) throw new GridClosureException(new IgniteException("Snapshot metafile must not exist: " + smf.getAbsolutePath())); @@ -966,7 +975,7 @@ private IgniteInternalFuture initLocalSnapshotEndStag try { if (req.error() != null) - deleteSnapshot(snapshotLocalDir(req.snapshotName()), pdsSettings.folderName()); + deleteSnapshot(snapshotLocalDir(req), pdsSettings.folderName()); removeLastMetaStorageKey(); } @@ -1416,6 +1425,15 @@ public List readSnapshotMetadatas(String snpName) { /** {@inheritDoc} */ @Override public IgniteFuture createSnapshot(String name) { + return createSnapshot(name, null); + } + + /** + * @param name + * @param snpPath + * @return + */ + public IgniteFuture createSnapshot(String name, @Nullable File snpPath) { A.notNullOrEmpty(name, "Snapshot name cannot be null or empty."); A.ensure(U.alphanumericUnderscore(name), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_"); @@ -1505,7 +1523,8 @@ public List readSnapshotMetadatas(String snpName) { grps, new HashSet<>(F.viewReadOnly(srvNodes, F.node2id(), - (node) -> CU.baselineNode(node, clusterState))))); + (node) -> CU.baselineNode(node, clusterState))), + snpPath == null ? null : snpPath.toString())); String msg = "Cluster-wide snapshot operation started [snpName=" + name + ", grps=" + grps + ']'; @@ -1865,14 +1884,14 @@ private SnapshotFutureTask currentSnapshotTask() { /** * @param factory Factory which produces {@link LocalSnapshotSender} implementation. */ - void localSnapshotSenderFactory(Function factory) { + void localSnapshotSenderFactory(BiFunction factory) { locSndrFactory = factory; } /** * @return Factory which produces {@link LocalSnapshotSender} implementation. */ - Function localSnapshotSenderFactory() { + BiFunction localSnapshotSenderFactory() { return locSndrFactory; } @@ -3074,13 +3093,15 @@ private class LocalSnapshotSender extends SnapshotSender { private final int pageSize; /** + * @param snpPath Snapshot name. * @param snpName Snapshot name. */ - public LocalSnapshotSender(String snpName) { + public LocalSnapshotSender(String snpName, @Nullable File snpPath) { super(IgniteSnapshotManager.this.log, cctx.kernalContext().pools().getSnapshotExecutorService()); this.snpName = snpName; - snpLocDir = snapshotLocalDir(snpName); + + snpLocDir = snpPath == null ? snapshotLocalDir(snpName) : new File(snpPath, snpName); pageSize = cctx.kernalContext().config().getDataStorageConfiguration().getPageSize(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java index 4cbcb5783f9848..c7b17de8dc4265 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; +import java.io.File; import java.io.Serializable; import java.util.Collection; import java.util.Set; @@ -40,6 +41,9 @@ public class SnapshotOperationRequest implements Serializable { /** Snapshot name. */ private final String snpName; + /** Snapshot path. */ + private final String snpPath; + /** Baseline node IDs that must be alive to complete the operation. */ @GridToStringInclude private final Set nodes; @@ -63,19 +67,22 @@ public class SnapshotOperationRequest implements Serializable { * @param snpName Snapshot name. * @param grps List of cache group names. * @param nodes Baseline node IDs that must be alive to complete the operation. + * @param snpPath Snapshot path. */ public SnapshotOperationRequest( UUID reqId, UUID opNodeId, String snpName, @Nullable Collection grps, - Set nodes + Set nodes, + @Nullable String snpPath ) { this.reqId = reqId; this.opNodeId = opNodeId; this.snpName = snpName; this.grps = grps; this.nodes = nodes; + this.snpPath = snpPath; } /** @@ -92,6 +99,13 @@ public String snapshotName() { return snpName; } + /** + * @return Snapshot path. + */ + public File snapshotPath() { + return snpPath == null ? null : new File(snpPath); + } + /** * @return List of cache group names. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 01489646426981..c2637d02bb6c4f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -331,8 +331,8 @@ public IgniteFuture start(String snpName, @Nullable Collection cac Collection bltNodes = F.viewReadOnly(ctx.discovery().discoCache().aliveBaselineNodes(), F.node2id()); - SnapshotOperationRequest req = - new SnapshotOperationRequest(fut0.rqId, F.first(dataNodes), snpName, cacheGrpNames, new HashSet<>(bltNodes)); + SnapshotOperationRequest req = new SnapshotOperationRequest( + fut0.rqId, F.first(dataNodes), snpName, cacheGrpNames, new HashSet<>(bltNodes), null); prepareRestoreProc.start(req.requestId(), req); }); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java index 5a9551374ec279..1212e410b4be63 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java @@ -562,13 +562,13 @@ protected static List setBlockingSnapshotExecutor(List old = mgr.localSnapshotSenderFactory(); + BiFunction old = mgr.localSnapshotSenderFactory(); BlockingExecutor block = new BlockingExecutor(mgr.snapshotExecutorService()); execs.add(block); - mgr.localSnapshotSenderFactory((snpName) -> - new DelegateSnapshotSender(log, block, old.apply(snpName))); + mgr.localSnapshotSenderFactory((snpName, snpPath) -> + new DelegateSnapshotSender(log, block, old.apply(snpName, snpPath))); } return execs; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java index 7e8421ce6d6641..e4ebdeca113d05 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java @@ -284,7 +284,7 @@ public void testSnapshotTaskIsBlockedWithoutMetastore() throws Exception { GridTestUtils.assertThrowsAnyCause(log, () -> snp(ig).registerSnapshotTask(SNAPSHOT_NAME, ig.localNode().id(), F.asMap(CU.cacheId(dfltCacheCfg.getName()), null), false, - snp(ig).localSnapshotSenderFactory().apply(SNAPSHOT_NAME)).get(TIMEOUT), + snp(ig).localSnapshotSenderFactory().apply(SNAPSHOT_NAME, null)).get(TIMEOUT), IgniteCheckedException.class, "Metastore is required because it holds encryption keys"); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java index 170774daa402f3..e201d58bd73d69 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java @@ -35,6 +35,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Predicate; import org.apache.ignite.Ignite; @@ -711,7 +712,18 @@ public void testClusterSnapshotWithRebalancing() throws Exception { /** @throws Exception If fails. */ @Test public void testClusterSnapshotWithExplicitPath() throws Exception { - File exSnpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); + doTestClusterSnapshotWithExplicitPath(true); + doTestClusterSnapshotWithExplicitPath(false); + } + + /** + * @param cfgPath {@code True} to set destination path using configuration, {@code False} using runtime parameter. + * @throws Exception If failed. + */ + private void doTestClusterSnapshotWithExplicitPath(boolean cfgPath) throws Exception { + File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); + + assert snpDir.list().length == 0 : "Target directory is not empty: " + Arrays.asList(snpDir.list()); try { IgniteEx ignite = null; @@ -719,7 +731,8 @@ public void testClusterSnapshotWithExplicitPath() throws Exception { for (int i = 0; i < 2; i++) { IgniteConfiguration cfg = optimize(getConfiguration(getTestIgniteInstanceName(i))); - cfg.setSnapshotPath(exSnpDir.getAbsolutePath()); + if (cfgPath) + cfg.setSnapshotPath(snpDir.getAbsolutePath()); ignite = startGrid(cfg); } @@ -730,19 +743,18 @@ public void testClusterSnapshotWithExplicitPath() throws Exception { for (int i = 0; i < CACHE_KEYS_RANGE; i++) ignite.cache(DEFAULT_CACHE_NAME).put(i, i); - ignite.snapshot().createSnapshot(SNAPSHOT_NAME) - .get(); + ignite.context().cache().context().snapshotMgr().createSnapshot(SNAPSHOT_NAME, cfgPath ? null : snpDir).get(); stopAllGrids(); - IgniteEx snp = startGridsFromSnapshot(2, cfg -> exSnpDir.getAbsolutePath(), SNAPSHOT_NAME, true); + IgniteEx snp = startGridsFromSnapshot(2, cfg -> snpDir.getAbsolutePath(), SNAPSHOT_NAME, true); assertSnapshotCacheKeys(snp.cache(dfltCacheCfg.getName())); } finally { stopAllGrids(); - U.delete(exSnpDir); + U.delete(snpDir); } } @@ -1094,12 +1106,12 @@ public void testConcurrentClusterSnapshotFromClient() throws Exception { IgniteEx clnt = startClientGrid(2); IgniteSnapshotManager mgr = snp(grid); - Function old = mgr.localSnapshotSenderFactory(); + BiFunction old = mgr.localSnapshotSenderFactory(); BlockingExecutor block = new BlockingExecutor(mgr.snapshotExecutorService()); - mgr.localSnapshotSenderFactory((snpName) -> - new DelegateSnapshotSender(log, block, old.apply(snpName))); + mgr.localSnapshotSenderFactory((snpName, snpPath) -> + new DelegateSnapshotSender(log, block, old.apply(snpName, snpPath))); IgniteFuture fut = grid.snapshot().createSnapshot(SNAPSHOT_NAME); @@ -1160,13 +1172,13 @@ public void testClusterSnapshotFinishedTryCancel() throws Exception { * @param blocked Latch to await delta partition processing. * @return Factory which produces local snapshot senders. */ - private Function blockingLocalSnapshotSender(IgniteEx ignite, + private BiFunction blockingLocalSnapshotSender(IgniteEx ignite, CountDownLatch started, CountDownLatch blocked ) { - Function old = snp(ignite).localSnapshotSenderFactory(); + BiFunction old = snp(ignite).localSnapshotSenderFactory(); - return (snpName) -> new DelegateSnapshotSender(log, snp(ignite).snapshotExecutorService(), old.apply(snpName)) { + return (snpName, snpPath) -> new DelegateSnapshotSender(log, snp(ignite).snapshotExecutorService(), old.apply(snpName, null)) { @Override public void sendDelta0(File delta, String cacheDirName, GroupPartitionId pair) { if (log.isInfoEnabled()) log.info("Processing delta file has been blocked: " + delta.getName()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManagerSelfTest.java index 60bf32fd2fbb8e..ee34e16da4ee4d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManagerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManagerSelfTest.java @@ -146,7 +146,7 @@ public void testSnapshotLocalPartitionMultiCpWithLoad() throws Exception { cctx.localNodeId(), F.asMap(CU.cacheId(DEFAULT_CACHE_NAME), null), encryption, - new DelegateSnapshotSender(log, mgr.snapshotExecutorService(), mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME)) { + new DelegateSnapshotSender(log, mgr.snapshotExecutorService(), mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME, null)) { @Override public void sendPart0(File part, String cacheDirName, GroupPartitionId pair, Long length) { try { U.await(slowCopy); @@ -268,7 +268,7 @@ public void testSnapshotLocalPartitionNotEnoughSpace() throws Exception { SNAPSHOT_NAME, F.asMap(CU.cacheId(DEFAULT_CACHE_NAME), null), encryption, - mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME)); + mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME, null)); // Check the right exception thrown. assertThrowsAnyCause(log, @@ -293,7 +293,7 @@ public void testSnapshotCreateLocalCopyPartitionFail() throws Exception { parts, encryption, new DelegateSnapshotSender(log, mgr0.snapshotExecutorService(), - mgr0.localSnapshotSenderFactory().apply(SNAPSHOT_NAME)) { + mgr0.localSnapshotSenderFactory().apply(SNAPSHOT_NAME, null)) { @Override public void sendPart0(File part, String cacheDirName, GroupPartitionId pair, Long length) { if (pair.getPartitionId() == 0) throw new IgniteException(err_msg + pair); @@ -328,7 +328,7 @@ public void testLocalSnapshotOnCacheStopped() throws Exception { SNAPSHOT_NAME, F.asMap(CU.cacheId(DEFAULT_CACHE_NAME), null), encryption, - new DelegateSnapshotSender(log, mgr.snapshotExecutorService(), mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME)) { + new DelegateSnapshotSender(log, mgr.snapshotExecutorService(), mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME, null)) { @Override public void sendPart0(File part, String cacheDirName, GroupPartitionId pair, Long length) { try { U.await(cpLatch); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/PlainSnapshotTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/PlainSnapshotTest.java index 21cdfdb3311bf0..3e0a32c3c267d1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/PlainSnapshotTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/PlainSnapshotTest.java @@ -85,7 +85,7 @@ public void testSnapshotLocalPartitions() throws Exception { IgniteInternalFuture snpFut = startLocalSnapshotTask(cctx, SNAPSHOT_NAME, F.asMap(CU.cacheId(DEFAULT_CACHE_NAME), null), - false, mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME)); + false, mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME, null)); snpFut.get(); From b285df92a6f5eea964514de0df21ea79d3d6134d Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Wed, 25 May 2022 18:42:49 +0300 Subject: [PATCH 03/19] wip --- .../AbstractSnapshotVerificationTask.java | 7 +- .../snapshot/IgniteSnapshotManager.java | 67 ++++++++++---- .../snapshot/SnapshotHandlerContext.java | 15 +++- .../snapshot/SnapshotHandlerRestoreTask.java | 22 +++-- .../SnapshotMetadataCollectorTask.java | 8 +- .../SnapshotMetadataCollectorTaskArg.java | 88 +++++++++++++++++++ .../snapshot/SnapshotOperationRequest.java | 4 +- .../SnapshotPartitionsVerifyHandler.java | 4 +- .../SnapshotPartitionsVerifyTask.java | 15 +++- .../SnapshotPartitionsVerifyTaskArg.java | 25 +++++- .../snapshot/SnapshotRestoreProcess.java | 19 ++-- .../IgniteClusterSnapshotCheckTest.java | 4 +- .../IgniteClusterSnapshotRestoreSelfTest.java | 56 ++++++++++++ 13 files changed, 287 insertions(+), 47 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java index addf246f954a6c..911ecf0620d201 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; +import java.io.File; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -33,6 +34,7 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.resources.IgniteInstanceResource; +import org.jetbrains.annotations.Nullable; /** * The task for checking the consistency of snapshots in the cluster. @@ -80,7 +82,7 @@ public abstract class AbstractSnapshotVerificationTask extends if (meta == null) continue; - jobs.put(createJob(meta.snapshotName(), meta.consistentId(), arg.cacheGroupNames()), + jobs.put(createJob(meta.snapshotName(), arg.snapshotPath(), meta.consistentId(), arg.cacheGroupNames()), e.getKey()); if (allMetas.isEmpty()) @@ -122,9 +124,10 @@ public static void checkMissedMetadata(Collection clusterMetas /** * @param name Snapshot name. + * @param snpPath todo * @param constId Snapshot metadata file name. * @param groups Cache groups to be restored from the snapshot. May be empty if all cache groups are being restored. * @return Compute job. */ - protected abstract ComputeJob createJob(String name, String constId, Collection groups); + protected abstract ComputeJob createJob(String name, @Nullable File snpPath, String constId, Collection groups); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 47b12a412ae581..9f5f8a3626c8a1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -714,8 +714,8 @@ public File snapshotLocalDir(String snpName) { * @return Local snapshot directory for snapshot with given name. * @throws IgniteCheckedException If directory doesn't exist. */ - private File resolveSnapshotDir(String snpName) throws IgniteCheckedException { - File snpDir = snapshotLocalDir(snpName); + private File resolveSnapshotDir(String snpName, @Nullable File snpPath) throws IgniteCheckedException { + File snpDir = snpPath == null ? snapshotLocalDir(snpName) : snpPath; if (!snpDir.exists()) throw new IgniteCheckedException("Snapshot directory doesn't exists: " + snpDir.getAbsolutePath()); @@ -817,12 +817,14 @@ private IgniteInternalFuture initLocalSnapshotStartSt .map(n -> cctx.discovery().node(n).consistentId().toString()) .collect(Collectors.toSet()); - File smf = new File(snapshotLocalDir(req), snapshotMetaFileName(cctx.localNode().consistentId().toString())); + File snpPath = snapshotLocalDir(req); + + File smf = new File(snpPath, snapshotMetaFileName(cctx.localNode().consistentId().toString())); if (smf.exists()) throw new GridClosureException(new IgniteException("Snapshot metafile must not exist: " + smf.getAbsolutePath())); - smf.getParentFile().mkdirs(); + snpPath.mkdirs(); SnapshotMetadata meta = new SnapshotMetadata(req.requestId(), req.snapshotName(), @@ -841,7 +843,8 @@ private IgniteInternalFuture initLocalSnapshotStartSt log.info("Snapshot metafile has been created: " + smf.getAbsolutePath()); } - SnapshotHandlerContext ctx = new SnapshotHandlerContext(meta, req.groups(), cctx.localNode()); + SnapshotHandlerContext ctx = + new SnapshotHandlerContext(meta, req.groups(), cctx.localNode(), snpPath.getParentFile()); return new SnapshotOperationResponse(handlers.invokeAll(SnapshotHandlerType.CREATE, ctx)); } @@ -1188,7 +1191,7 @@ public IgniteInternalFuture checkSnapshot(String name) { cctx.kernalContext().security().authorize(ADMIN_SNAPSHOT); - return checkSnapshot(name, null, false).chain(f -> { + return checkSnapshot(name, null, null, false).chain(f -> { try { return f.get().idleVerifyResult(); } @@ -1212,6 +1215,7 @@ public IgniteInternalFuture checkSnapshot(String name) { */ public IgniteInternalFuture checkSnapshot( String name, + @Nullable File snpPath, @Nullable Collection grps, boolean includeCustomHandlers ) { @@ -1230,7 +1234,9 @@ public IgniteInternalFuture checkSnapshot( kctx0.task().setThreadContext(TC_SKIP_AUTH, true); kctx0.task().setThreadContext(TC_SUBGRID, bltNodes); - kctx0.task().execute(SnapshotMetadataCollectorTask.class, name).listen(f0 -> { + SnapshotMetadataCollectorTaskArg taskArg = new SnapshotMetadataCollectorTaskArg(name, snpPath); + + kctx0.task().execute(SnapshotMetadataCollectorTask.class, taskArg).listen(f0 -> { if (f0.error() == null) { Map> metas = f0.result(); @@ -1275,13 +1281,22 @@ public IgniteInternalFuture checkSnapshot( return; } + if (metas.isEmpty()) { + res.onDone(new SnapshotPartitionsVerifyTaskResult(metas, + new IdleVerifyResultV2(Collections.singletonMap(cctx.localNode(), + new IllegalArgumentException("Snapshot does not exists [snapshot=" + name + + (snpPath != null ? ", baseDir=" + snpPath : "") + ']'))))); + + return; + } + kctx0.task().setThreadContext(TC_SKIP_AUTH, true); kctx0.task().setThreadContext(TC_SUBGRID, new ArrayList<>(metas.keySet())); Class cls = includeCustomHandlers ? SnapshotHandlerRestoreTask.class : SnapshotPartitionsVerifyTask.class; - kctx0.task().execute(cls, new SnapshotPartitionsVerifyTaskArg(grps, metas)) + kctx0.task().execute(cls, new SnapshotPartitionsVerifyTaskArg(grps, metas, snpPath)) .listen(f1 -> { if (f1.error() == null) res.onDone(f1.result()); @@ -1310,17 +1325,21 @@ else if (f1.error() instanceof IgniteSnapshotVerifyException) * @return The list of cache or cache group names in given snapshot on local node. */ public List snapshotCacheDirectories(String snpName, String folderName) { - return snapshotCacheDirectories(snpName, folderName, name -> true); + return snapshotCacheDirectories(snpName, null, folderName, name -> true); } +// public List snapshotCacheDirectories(String snpName, @Nullable File snpPath, String folderName) { +// return snapshotCacheDirectories(snpName, snpPath, folderName, name -> true); +// } + /** * @param snpName Snapshot name. * @param folderName The name of a directory for the cache group. * @param names Cache group names to filter. * @return The list of cache or cache group names in given snapshot on local node. */ - public List snapshotCacheDirectories(String snpName, String folderName, Predicate names) { - File snpDir = snapshotLocalDir(snpName); + public List snapshotCacheDirectories(String snpName, @Nullable File snpPath, String folderName, Predicate names) { + File snpDir = snpPath == null ? snapshotLocalDir(snpName) : new File(snpPath, snpName); if (!snpDir.exists()) return Collections.emptyList(); @@ -1334,7 +1353,11 @@ public List snapshotCacheDirectories(String snpName, String folderName, Pr * @return Snapshot metadata instance. */ public SnapshotMetadata readSnapshotMetadata(String snpName, String consId) { - return readSnapshotMetadata(new File(snapshotLocalDir(snpName), snapshotMetaFileName(consId))); + return readSnapshotMetadata(snpName, null, consId); + } + + public SnapshotMetadata readSnapshotMetadata(String snpName, @Nullable File snpDir, String consId) { + return readSnapshotMetadata(new File(snpDir == null ? snapshotLocalDir(snpName) : snpDir, snapshotMetaFileName(consId))); } /** @@ -1371,10 +1394,14 @@ private SnapshotMetadata readSnapshotMetadata(File smf) { * local node will be placed on the first place. */ public List readSnapshotMetadatas(String snpName) { + return readSnapshotMetadatas(snpName, null); + } + + public List readSnapshotMetadatas(String snpName, @Nullable File snpPath) { A.notNullOrEmpty(snpName, "Snapshot name cannot be null or empty."); A.ensure(U.alphanumericUnderscore(snpName), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_"); - File snpDir = snapshotLocalDir(snpName); + File snpDir = snpPath == null ? snapshotLocalDir(snpName) : new File(snpPath, snpName); if (!(snpDir.exists() && snpDir.isDirectory())) return Collections.emptyList(); @@ -1524,7 +1551,7 @@ public IgniteFuture createSnapshot(String name, @Nullable File snpPath) { new HashSet<>(F.viewReadOnly(srvNodes, F.node2id(), (node) -> CU.baselineNode(node, clusterState))), - snpPath == null ? null : snpPath.toString())); + snpPath)); String msg = "Cluster-wide snapshot operation started [snpName=" + name + ", grps=" + grps + ']'; @@ -1548,11 +1575,15 @@ public IgniteFuture createSnapshot(String name, @Nullable File snpPath) { /** {@inheritDoc} */ @Override public IgniteFuture restoreSnapshot(String name, @Nullable Collection grpNames) { + return restoreSnapshot(name, grpNames, null); + } + + public IgniteFuture restoreSnapshot(String name, @Nullable Collection grpNames, @Nullable File snpPath) { A.notNullOrEmpty(name, "Snapshot name cannot be null or empty."); A.ensure(U.alphanumericUnderscore(name), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_"); A.ensure(grpNames == null || !grpNames.isEmpty(), "List of cache group names cannot be empty."); - return restoreCacheGrpProc.start(name, grpNames); + return restoreCacheGrpProc.start(name, grpNames, snpPath); } /** {@inheritDoc} */ @@ -1696,8 +1727,8 @@ private static String snapshotMetaFileName(String consId) { * @return Standalone kernal context related to the snapshot. * @throws IgniteCheckedException If fails. */ - public StandaloneGridKernalContext createStandaloneKernalContext(String snpName, String folderName) throws IgniteCheckedException { - File snpDir = resolveSnapshotDir(snpName); + public StandaloneGridKernalContext createStandaloneKernalContext(String snpName, File snpPath, String folderName) throws IgniteCheckedException { + File snpDir = resolveSnapshotDir(snpName, snpPath); return new StandaloneGridKernalContext(log, resolveBinaryWorkDir(snpDir.getAbsolutePath(), folderName), @@ -1742,7 +1773,7 @@ public GridCloseableIterator partitionRowIterator(String snpName, int partId, @Nullable EncryptionCacheKeyProvider encrKeyProvider ) throws IgniteCheckedException { - File snpDir = resolveSnapshotDir(snpName); + File snpDir = resolveSnapshotDir(snpName, null); File nodePath = new File(snpDir, databaseRelativePath(folderName)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerContext.java index 3cdf7153150d38..73612aae44f18f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerContext.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; +import java.io.File; import java.util.Collection; import org.apache.ignite.cluster.ClusterNode; import org.jetbrains.annotations.Nullable; @@ -28,6 +29,9 @@ public class SnapshotHandlerContext { /** Snapshot metadata. */ private final SnapshotMetadata metadata; + /** Snapshot directory path. */ + private final String snpBasePath; + /** The names of the cache groups on which the operation is performed. */ private final Collection grps; @@ -38,11 +42,13 @@ public class SnapshotHandlerContext { * @param metadata Snapshot metadata. * @param grps The names of the cache groups on which the operation is performed. * @param locNode Local node. + * @param snpBasePath Snapshot directory path. */ - public SnapshotHandlerContext(SnapshotMetadata metadata, @Nullable Collection grps, ClusterNode locNode) { + public SnapshotHandlerContext(SnapshotMetadata metadata, @Nullable Collection grps, ClusterNode locNode, File snpBasePath) { this.metadata = metadata; this.grps = grps; this.locNode = locNode; + this.snpBasePath = snpBasePath.toString(); } /** @@ -52,6 +58,13 @@ public SnapshotMetadata metadata() { return metadata; } + /** + * @return Snapshot directory path. + */ + public File snapshotPath() { + return new File(snpBasePath); + } + /** * @return The names of the cache groups on which the operation is performed. May be {@code null} if the operation * is performed on all available cache groups. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java index 8db322c90481f7..1e5f376d1fefb2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; +import java.io.File; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -47,8 +48,13 @@ public class SnapshotHandlerRestoreTask extends AbstractSnapshotVerificationTask private IgniteLogger log; /** {@inheritDoc} */ - @Override protected ComputeJob createJob(String snpName, String constId, Collection groups) { - return new SnapshotHandlerRestoreJob(snpName, constId, groups); + @Override protected ComputeJob createJob( + String snpName, + @Nullable File snpPath, + String constId, + Collection groups + ) { + return new SnapshotHandlerRestoreJob(snpName, snpPath, constId, groups); } /** {@inheritDoc} */ @@ -112,23 +118,29 @@ private static class SnapshotHandlerRestoreJob extends ComputeJobAdapter { /** Cache group names. */ private final Collection grps; + private final File snpPath; + /** * @param snpName Snapshot name. + * @param snpPath todo * @param consistentId String representation of the consistent node ID. * @param grps Cache group names. */ - public SnapshotHandlerRestoreJob(String snpName, String consistentId, Collection grps) { + public SnapshotHandlerRestoreJob(String snpName, @Nullable File snpPath, String consistentId, Collection grps) { this.snpName = snpName; this.consistentId = consistentId; this.grps = grps; + this.snpPath = snpPath; } /** {@inheritDoc} */ @Override public Map> execute() { try { IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); - SnapshotMetadata meta = snpMgr.readSnapshotMetadata(snpName, consistentId); - SnapshotHandlerContext ctx = new SnapshotHandlerContext(meta, grps, ignite.localNode()); + File snpDir = snpPath == null ? snpMgr.snapshotLocalDir(snpName) : new File(snpPath, snpName); + SnapshotMetadata meta = snpMgr.readSnapshotMetadata(snpName, snpDir, consistentId); + + SnapshotHandlerContext ctx = new SnapshotHandlerContext(meta, grps, ignite.localNode(), snpDir.getParentFile()); return snpMgr.handlers().invokeAll(SnapshotHandlerType.RESTORE, ctx); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java index c2c3b5c4ba2324..76fb7b31750c8d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java @@ -38,25 +38,25 @@ /** Snapshot task to collect snapshot metadata from the baseline nodes for given snapshot name. */ @GridInternal public class SnapshotMetadataCollectorTask - extends ComputeTaskAdapter>> { + extends ComputeTaskAdapter>> { /** Serial version uid. */ private static final long serialVersionUID = 0L; /** {@inheritDoc} */ @Override public @NotNull Map map( List subgrid, - @Nullable String snpName + SnapshotMetadataCollectorTaskArg arg ) throws IgniteException { Map map = U.newHashMap(subgrid.size()); for (ClusterNode node : subgrid) { - map.put(new ComputeJobAdapter(snpName) { + map.put(new ComputeJobAdapter() { @IgniteInstanceResource private transient IgniteEx ignite; @Override public List execute() throws IgniteException { return ignite.context().cache().context().snapshotMgr() - .readSnapshotMetadatas(snpName); + .readSnapshotMetadatas(arg.snapshotName(), arg.snapshotPath()); } }, node); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java new file mode 100644 index 00000000000000..9e25a546119488 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.VisorDataTransferObject; + +/** + * Input parameters for checking snapshot partitions consistency task. + */ +public class SnapshotMetadataCollectorTaskArg extends VisorDataTransferObject { + /** Serial version UID. */ + private static final long serialVersionUID = 0L; + + /** Snapshot name. */ + private String snpName; + + /** Snapshot directory path. */ + private File snpPath; + + /** Default constructor. */ + public SnapshotMetadataCollectorTaskArg() { + // No-op. + } + + /** + * @param snpName Snapshot name. + * @param snpPath SNapshot directory path. + */ + public SnapshotMetadataCollectorTaskArg(String snpName, File snpPath) { + this.snpName = snpName; + this.snpPath = snpPath; + } + + /** + * @return Snapshot name. + */ + public String snapshotName() { + return snpName; + } + + /** + * @return Snapshot directory path. + */ + public File snapshotPath() { + return snpPath; + } + + /** {@inheritDoc} */ + @Override protected void writeExternalData(ObjectOutput out) throws IOException { + U.writeString(out, snpName); + U.writeString(out, snpPath == null ? null : snpPath.toString()); + } + + /** {@inheritDoc} */ + @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { + snpName = U.readString(in); + + String path = U.readString(in); + + if (path != null) + snpPath = new File(path); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(SnapshotMetadataCollectorTaskArg.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java index c7b17de8dc4265..5c86f59c8f7cce 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java @@ -75,14 +75,14 @@ public SnapshotOperationRequest( String snpName, @Nullable Collection grps, Set nodes, - @Nullable String snpPath + @Nullable File snpPath ) { this.reqId = reqId; this.opNodeId = opNodeId; this.snpName = snpName; this.grps = grps; this.nodes = nodes; - this.snpPath = snpPath; + this.snpPath = snpPath == null ? null : snpPath.toString(); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java index f6c2b78f3a0f57..2d0ee271a722fe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java @@ -109,7 +109,7 @@ public SnapshotPartitionsVerifyHandler(GridCacheSharedContext cctx) { IgniteSnapshotManager snpMgr = cctx.snapshotMgr(); - for (File dir : snpMgr.snapshotCacheDirectories(meta.snapshotName(), meta.folderName())) { + for (File dir : snpMgr.snapshotCacheDirectories(meta.snapshotName(), opCtx.snapshotPath(), meta.folderName(), name -> true)) { int grpId = CU.cacheId(cacheGroupName(dir)); if (!grps.remove(grpId)) @@ -146,7 +146,7 @@ public SnapshotPartitionsVerifyHandler(GridCacheSharedContext cctx) { ThreadLocal buff = ThreadLocal.withInitial(() -> ByteBuffer.allocateDirect(meta.pageSize()) .order(ByteOrder.nativeOrder())); - GridKernalContext snpCtx = snpMgr.createStandaloneKernalContext(meta.snapshotName(), meta.folderName()); + GridKernalContext snpCtx = snpMgr.createStandaloneKernalContext(meta.snapshotName(), opCtx.snapshotPath(), meta.folderName()); FilePageStoreManager storeMgr = (FilePageStoreManager)cctx.pageStore(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java index 040802f9fc7673..ea6e1ff5514a77 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; +import java.io.File; import java.util.Collection; import java.util.List; import java.util.Map; @@ -54,8 +55,8 @@ public class SnapshotPartitionsVerifyTask extends AbstractSnapshotVerificationTa private IgniteEx ignite; /** {@inheritDoc} */ - @Override protected ComputeJob createJob(String name, String constId, Collection groups) { - return new VisorVerifySnapshotPartitionsJob(name, constId, groups); + @Override protected ComputeJob createJob(String name, File snpPath, String constId, Collection groups) { + return new VisorVerifySnapshotPartitionsJob(name, snpPath, constId, groups); } /** {@inheritDoc} */ @@ -85,15 +86,20 @@ private static class VisorVerifySnapshotPartitionsJob extends ComputeJobAdapter /** Set of cache groups to be checked in the snapshot or {@code empty} to check everything. */ private final Collection rqGrps; + /** Snapshot directory path. */ + private final File snpPath; + /** * @param snpName Snapshot name to validate. * @param consId Consistent snapshot metadata file name. * @param rqGrps Set of cache groups to be checked in the snapshot or {@code empty} to check everything. + * @param snpPath Snapshot directory path. */ - public VisorVerifySnapshotPartitionsJob(String snpName, String consId, Collection rqGrps) { + public VisorVerifySnapshotPartitionsJob(String snpName, @Nullable File snpPath, String consId, Collection rqGrps) { this.snpName = snpName; this.consId = consId; this.rqGrps = rqGrps; + this.snpPath = snpPath; } /** {@inheritDoc} */ @@ -107,9 +113,10 @@ public VisorVerifySnapshotPartitionsJob(String snpName, String consId, Collectio try { SnapshotMetadata meta = cctx.snapshotMgr().readSnapshotMetadata(snpName, consId); + File snpDir = snpPath == null ? cctx.snapshotMgr().snapshotLocalDir(snpName) : new File(snpPath, snpName); return new SnapshotPartitionsVerifyHandler(cctx) - .invoke(new SnapshotHandlerContext(meta, rqGrps, ignite.localNode())); + .invoke(new SnapshotHandlerContext(meta, rqGrps, ignite.localNode(), snpDir.getParentFile())); } catch (IgniteCheckedException e) { throw new IgniteException(e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTaskArg.java index 64a2426339ad5a..2f54a317603d96 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTaskArg.java @@ -16,6 +16,7 @@ */ package org.apache.ignite.internal.processors.cache.persistence.snapshot; +import java.io.File; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; @@ -26,6 +27,7 @@ import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorDataTransferObject; +import org.jetbrains.annotations.Nullable; /** * Input parameters for checking snapshot partitions consistency task. @@ -40,6 +42,8 @@ public class SnapshotPartitionsVerifyTaskArg extends VisorDataTransferObject { /** The map of distribution of snapshot metadata pieces across the cluster. */ private Map> clusterMetas; + private File snpPath; + /** Default constructor. */ public SnapshotPartitionsVerifyTaskArg() { // No-op. @@ -48,10 +52,16 @@ public SnapshotPartitionsVerifyTaskArg() { /** * @param grpNames Cache group names to be verified. * @param clusterMetas The map of distribution of snapshot metadata pieces across the cluster. + * @param snpPath todo */ - public SnapshotPartitionsVerifyTaskArg(Collection grpNames, Map> clusterMetas) { + public SnapshotPartitionsVerifyTaskArg( + Collection grpNames, + Map> clusterMetas, + @Nullable File snpPath + ) { this.grpNames = grpNames; this.clusterMetas = clusterMetas; + this.snpPath = snpPath; } /** @@ -68,16 +78,29 @@ public Map> clusterMetadata() { return clusterMetas; } + /** + * @return Snapshot directory path. + */ + public File snapshotPath() { + return snpPath; + } + /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeCollection(out, grpNames); U.writeMap(out, clusterMetas); + U.writeString(out, snpPath == null ? null : snpPath.toString()); } /** {@inheritDoc} */ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { grpNames = U.readCollection(in); clusterMetas = U.readMap(in); + + String snpPathname = U.readString(in); + + if (snpPathname != null) + snpPath = new File(snpPathname); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index c2637d02bb6c4f..a64ea33302ffa1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -211,7 +211,7 @@ protected void registerMetrics() { * @param cacheGrpNames Cache groups to be restored or {@code null} to restore all cache groups from the snapshot. * @return Future that will be completed when the restore operation is complete and the cache groups are started. */ - public IgniteFuture start(String snpName, @Nullable Collection cacheGrpNames) { + public IgniteFuture start(String snpName, @Nullable Collection cacheGrpNames, @Nullable File snpPath) { IgniteSnapshotManager snpMgr = ctx.cache().context().snapshotMgr(); ClusterSnapshotFuture fut0; @@ -277,7 +277,7 @@ public IgniteFuture start(String snpName, @Nullable Collection cac snpMgr.recordSnapshotEvent(snpName, msg, EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED); - snpMgr.checkSnapshot(snpName, cacheGrpNames, true).listen(f -> { + snpMgr.checkSnapshot(snpName, snpPath, cacheGrpNames, true).listen(f -> { if (f.error() != null) { finishProcess(fut0.rqId, f.error()); @@ -332,7 +332,7 @@ public IgniteFuture start(String snpName, @Nullable Collection cac Collection bltNodes = F.viewReadOnly(ctx.discovery().discoCache().aliveBaselineNodes(), F.node2id()); SnapshotOperationRequest req = new SnapshotOperationRequest( - fut0.rqId, F.first(dataNodes), snpName, cacheGrpNames, new HashSet<>(bltNodes), null); + fut0.rqId, F.first(dataNodes), snpName, cacheGrpNames, new HashSet<>(bltNodes), snpPath); prepareRestoreProc.start(req.requestId(), req); }); @@ -582,7 +582,7 @@ private IgniteInternalFuture prepare(SnapshotO ", caches=" + req.groups() + ']'); } - List locMetas = snpMgr.readSnapshotMetadatas(req.snapshotName()); + List locMetas = snpMgr.readSnapshotMetadatas(req.snapshotName(), req.snapshotPath()); SnapshotRestoreContext opCtx0 = prepareContext(req, locMetas); @@ -666,7 +666,7 @@ private SnapshotRestoreContext prepareContext( // Collect the cache configurations and prepare a temporary directory for copying files. // Metastorage can be restored only manually by directly copying files. for (SnapshotMetadata meta : metas) { - for (File snpCacheDir : cctx.snapshotMgr().snapshotCacheDirectories(req.snapshotName(), meta.folderName(), + for (File snpCacheDir : cctx.snapshotMgr().snapshotCacheDirectories(req.snapshotName(), req.snapshotPath(), meta.folderName(), name -> !METASTORAGE_CACHE_NAME.equals(name))) { String grpName = FilePageStoreManager.cacheGroupName(snpCacheDir); @@ -916,7 +916,10 @@ private IgniteInternalFuture preload(UUID reqId) { if (leftParts.isEmpty()) break; - File snpCacheDir = new File(ctx.cache().context().snapshotMgr().snapshotLocalDir(opCtx0.snpName), + File snpDir = opCtx0.snpPath != null ? new File(opCtx0.snpPath, opCtx0.snpName) : + ctx.cache().context().snapshotMgr().snapshotLocalDir(opCtx0.snpName); + + File snpCacheDir = new File(snpDir, Paths.get(databaseRelativePath(meta.folderName()), dir.getName()).toString()); leftParts.removeIf(partFut -> { @@ -1383,6 +1386,8 @@ private static class SnapshotRestoreContext { /** Snapshot name. */ private final String snpName; + private final File snpPath; + /** Baseline discovery cache for node IDs that must be alive to complete the operation.*/ private final DiscoCache discoCache; @@ -1432,6 +1437,7 @@ protected SnapshotRestoreContext() { startTime = 0; opNodeId = null; discoCache = null; + snpPath = null; } /** @@ -1442,6 +1448,7 @@ protected SnapshotRestoreContext() { protected SnapshotRestoreContext(SnapshotOperationRequest req, DiscoCache discoCache, Map cfgs) { reqId = req.requestId(); snpName = req.snapshotName(); + snpPath = req.snapshotPath(); opNodeId = req.operationalNodeId(); startTime = U.currentTimeMillis(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java index 974fc8d11dfb2c..a3106d97205b69 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java @@ -478,7 +478,7 @@ public void testClusterSnapshotCheckHashesSameAsIdleVerifyHashes() throws Except IdleVerifyResultV2 snpVerifyRes = ignite.compute().execute(new TestSnapshotPartitionsVerifyTask(), new SnapshotPartitionsVerifyTaskArg(new HashSet<>(), Collections.singletonMap(ignite.cluster().localNode(), Collections.singletonList(snp(ignite).readSnapshotMetadata(SNAPSHOT_NAME, - (String)ignite.configuration().getConsistentId()))))) + (String)ignite.configuration().getConsistentId()))), null)) .idleVerifyResult(); Map> idleVerifyHashes = jobResults.get(TestVisorBackupPartitionsTask.class); @@ -599,7 +599,7 @@ private SnapshotPartitionsVerifyTaskResult checkSnapshotWithTwoCachesWhenOneIsCo corruptPartitionFile(ignite, SNAPSHOT_NAME, ccfg1, PART_ID); - return snp(ignite).checkSnapshot(SNAPSHOT_NAME, cachesToCheck, false).get(TIMEOUT); + return snp(ignite).checkSnapshot(SNAPSHOT_NAME, null, cachesToCheck, false).get(TIMEOUT); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index af70b8efe1c72b..8f09b0e7d115c7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -66,6 +66,7 @@ import org.jetbrains.annotations.Nullable; import org.junit.Test; +import static org.apache.ignite.cluster.ClusterState.ACTIVE; import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_FAILED; import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_FINISHED; import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED; @@ -116,6 +117,61 @@ public void testRestoreWithEmptyPartitions() throws Exception { assertCacheKeys(ignite.cache(DEFAULT_CACHE_NAME), keysCnt); } + @Test + public void testClusterSnapshotRestoreWithExplicitPath() throws Exception { + File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); + + assert snpDir.list().length == 0 : "Target directory is not empty: " + Arrays.asList(snpDir.list()); + + try { + IgniteEx ignite = null; + + for (int i = 0; i < 2; i++) + ignite = startGrid(i); +// IgniteConfiguration cfg = optimize(getConfiguration(getTestIgniteInstanceName(i))); +// +// if (cfgPath) +// cfg.setSnapshotPath(snpDir.getAbsolutePath()); +// +// ignite = startGrid(i); +// } + +// ignite.cluster().baselineAutoAdjustEnabled(false); + ignite.cluster().state(ACTIVE); + + for (int i = 0; i < CACHE_KEYS_RANGE; i++) + ignite.cache(DEFAULT_CACHE_NAME).put(i, i); + + IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); + + snpMgr.createSnapshot(SNAPSHOT_NAME, snpDir).get(); + +// snpMgr.checkSnapshot(SNAPSHOT_NAME).get(); + + ignite.destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + + snpMgr.restoreSnapshot(SNAPSHOT_NAME, null, snpDir).get(); + +// stopAllGrids(); + +// IgniteEx snp = startGridsFromSnapshot(2, cfg -> snpDir.getAbsolutePath(), SNAPSHOT_NAME, true); + + IgniteCache cache = ignite.cache(SNAPSHOT_NAME); + + assert cache != null; + + assertSnapshotCacheKeys(cache); + } + finally { + stopAllGrids(); + + U.delete(snpDir); + } + + } + /** * Ensures that system partition verification task is invoked before restoring the snapshot. * From 9b43eba83afca522f2abe270ef944520ba32753b Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Thu, 26 May 2022 16:47:27 +0300 Subject: [PATCH 04/19] IGNITE-15067 Request remote from custom location (wip). --- .../snapshot/IgniteSnapshotManager.java | 14 +++++- .../snapshot/SnapshotFilesRequestMessage.java | 36 ++++++++++++++- .../SnapshotMetadataCollectorTaskArg.java | 2 +- .../SnapshotResponseRemoteFutureTask.java | 12 +++-- .../snapshot/SnapshotRestoreProcess.java | 32 ++++++++++--- .../IgniteClusterSnapshotRestoreSelfTest.java | 46 ++++++++++++++----- .../IgniteSnapshotRemoteRequestTest.java | 10 ++-- 7 files changed, 124 insertions(+), 28 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 9f5f8a3626c8a1..247b15b7443807 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -1673,6 +1673,7 @@ public static boolean isSnapshotOperation(DiscoveryEvent evt) { public IgniteInternalFuture requestRemoteSnapshotFiles( UUID rmtNodeId, String snpName, + String snpPath, Map> parts, BooleanSupplier stopChecker, BiConsumer<@Nullable File, @Nullable Throwable> partHnd @@ -1690,7 +1691,8 @@ public IgniteInternalFuture requestRemoteSnapshotFiles( if (!nodeSupports(rmtNode, PERSISTENCE_CACHE_SNAPSHOT)) throw new IgniteCheckedException("Snapshot on remote node is not supported: " + rmtNode.id()); - RemoteSnapshotFilesRecevier fut = new RemoteSnapshotFilesRecevier(this, rmtNodeId, snpName, parts, stopChecker, partHnd); + RemoteSnapshotFilesRecevier fut = + new RemoteSnapshotFilesRecevier(this, rmtNodeId, snpName, snpPath, parts, stopChecker, partHnd); snpRmtMgr.submit(fut); @@ -2501,6 +2503,7 @@ private static class RemoteSnapshotFilesRecevier extends GridFutureAdapter * @param snpMgr Ignite snapshot manager. * @param rmtNodeId Remote node to request snapshot from. * @param snpName Snapshot name to request. + * @param rmptSnpPath Snapshot directory path. * @param parts Cache group and partitions to request. * @param stopChecker Process interrupt checker. * @param partHnd Partition handler. @@ -2509,12 +2512,13 @@ public RemoteSnapshotFilesRecevier( IgniteSnapshotManager snpMgr, UUID rmtNodeId, String snpName, + String rmtSnpPath, Map> parts, BooleanSupplier stopChecker, BiConsumer<@Nullable File, @Nullable Throwable> partHnd ) { dir = Paths.get(snpMgr.tmpWorkDir.getAbsolutePath(), reqId); - initMsg = new SnapshotFilesRequestMessage(reqId, snpName, parts); + initMsg = new SnapshotFilesRequestMessage(reqId, snpName, rmtSnpPath, parts); this.snpMgr = snpMgr; this.rmtNodeId = rmtNodeId; @@ -2748,10 +2752,14 @@ private Set activeTasks() { } } + File snpDir = reqMsg0.snapshotPath() == null ? + cctx.snapshotMgr().snapshotLocalDir(snpName) : new File(reqMsg0.snapshotPath(), snpName); + AbstractSnapshotFutureTask task = registerTask(rqId, new SnapshotResponseRemoteFutureTask(cctx, nodeId, snpName, + snpDir, tmpWorkDir, ioFactory, rmtSndrFactory.apply(rqId, nodeId), @@ -3047,6 +3055,8 @@ public RemoteSnapshotSender( assert len > 0 : "Requested partitions has incorrect file length " + "[pair=" + pair + ", cacheDirName=" + cacheDirName + ']'; + System.out.println(">xxx> send " + part); + sndr.send(part, 0, len, transmissionParams(rqId, cacheDirName, pair), TransmissionPolicy.FILE); if (log.isInfoEnabled()) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFilesRequestMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFilesRequestMessage.java index 65d13459eafc03..413d62e596a041 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFilesRequestMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFilesRequestMessage.java @@ -31,6 +31,7 @@ import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType; import org.apache.ignite.plugin.extensions.communication.MessageReader; import org.apache.ignite.plugin.extensions.communication.MessageWriter; +import org.jetbrains.annotations.Nullable; /** * @@ -45,6 +46,9 @@ public class SnapshotFilesRequestMessage extends AbstractSnapshotMessage { /** Snapshot name to request. */ private String snpName; + /** Snapshot directory path. */ + private String snpPath; + /** Map of cache group ids and corresponding set of its partition ids. */ @GridDirectMap(keyType = Integer.class, valueType = int[].class) private Map parts; @@ -59,14 +63,21 @@ public SnapshotFilesRequestMessage() { /** * @param reqId Unique request id. * @param snpName Snapshot name. + * @param snpPath Snapshot directory path. * @param parts Map of cache group ids and corresponding set of its partition ids to be snapshot. */ - public SnapshotFilesRequestMessage(String reqId, String snpName, Map> parts) { + public SnapshotFilesRequestMessage( + String reqId, + String snpName, + @Nullable String snpPath, + Map> parts + ) { super(reqId); assert parts != null && !parts.isEmpty(); this.snpName = snpName; + this.snpPath = snpPath; this.parts = new HashMap<>(); for (Map.Entry> e : parts.entrySet()) @@ -95,6 +106,13 @@ public String snapshotName() { return snpName; } + /** + * @return Snapshot directory path. + */ + public String snapshotPath() { + return snpPath; + } + /** {@inheritDoc} */ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) { writer.setBuffer(buf); @@ -123,6 +141,13 @@ public String snapshotName() { writer.incrementState(); } + if (writer.state() == 3) { + if (!writer.writeString("snpPath", snpPath)) + return false; + + writer.incrementState(); + } + return true; } @@ -154,6 +179,15 @@ public String snapshotName() { reader.incrementState(); } + if (reader.state() == 3) { + snpPath = reader.readString("snpPath"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + } + return reader.afterMessageRead(SnapshotFilesRequestMessage.class); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java index 9e25a546119488..d264ed37a1fd69 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java @@ -44,7 +44,7 @@ public SnapshotMetadataCollectorTaskArg() { /** * @param snpName Snapshot name. - * @param snpPath SNapshot directory path. + * @param snpPath Snapshot directory path. */ public SnapshotMetadataCollectorTaskArg(String snpName, File snpPath) { this.snpName = snpName; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java index 4763ce64e741ec..d79fb480076559 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java @@ -39,10 +39,14 @@ /** */ public class SnapshotResponseRemoteFutureTask extends AbstractSnapshotFutureTask { + /** */ + private final File snpDir; + /** * @param cctx Shared context. * @param srcNodeId Node id which cause snapshot task creation. * @param snpName Unique identifier of snapshot process. + * @param snpDir * @param tmpWorkDir Working directory for intermediate snapshot results. * @param ioFactory Factory to working with snapshot files. * @param snpSndr Factory which produces snapshot receiver instance. @@ -51,12 +55,15 @@ public SnapshotResponseRemoteFutureTask( GridCacheSharedContext cctx, UUID srcNodeId, String snpName, + File snpDir, File tmpWorkDir, FileIOFactory ioFactory, SnapshotSender snpSndr, Map> parts ) { super(cctx, srcNodeId, snpName, tmpWorkDir, ioFactory, snpSndr, parts); + + this.snpDir = snpDir; } /** {@inheritDoc} */ @@ -74,10 +81,9 @@ public SnapshotResponseRemoteFutureTask( snpSndr.init(handled.size()); - File snpDir = cctx.snapshotMgr().snapshotLocalDir(snpName); - List> futs = new ArrayList<>(); - List metas = cctx.snapshotMgr().readSnapshotMetadatas(snpName); + // todo + List metas = cctx.snapshotMgr().readSnapshotMetadatas(snpName, snpDir.getParentFile()); for (SnapshotMetadata meta : metas) { Map> parts0 = meta.partitions(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index a64ea33302ffa1..75a1b77c1f42e9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -303,6 +303,10 @@ public IgniteFuture start(String snpName, @Nullable Collection cac cacheGrpNames.stream().collect(Collectors.toMap(CU::cacheId, v -> v)); for (Map.Entry> entry : metas.entrySet()) { + // todo remove + if (entry.getKey().id().toString().endsWith("2")) + continue; + dataNodes.add(entry.getKey().id()); for (SnapshotMetadata meta : entry.getValue()) { @@ -582,7 +586,9 @@ private IgniteInternalFuture prepare(SnapshotO ", caches=" + req.groups() + ']'); } - List locMetas = snpMgr.readSnapshotMetadatas(req.snapshotName(), req.snapshotPath()); + List locMetas = snpMgr.readSnapshotMetadatas(req.snapshotName(), !ctx.localNodeId().toString().endsWith("2") ? req.snapshotPath() : null); + +// !ctx.localNodeId().toString().endsWith("2") ? req.snapshotPath() : null); SnapshotRestoreContext opCtx0 = prepareContext(req, locMetas); @@ -826,6 +832,9 @@ private IgniteInternalFuture preload(UUID reqId) { if (ctx.isStopping()) throw new NodeStoppingException("Node is stopping: " + ctx.localNodeId()); + File snpDir = opCtx0.snpPath != null ? new File(opCtx0.snpPath, opCtx0.snpName) : + ctx.cache().context().snapshotMgr().snapshotLocalDir(opCtx0.snpName); + Set allMetas = opCtx0.metasPerNode.values().stream().flatMap(List::stream).collect(Collectors.toSet()); @@ -850,8 +859,7 @@ private IgniteInternalFuture preload(UUID reqId) { try { SnapshotMetadata meta = F.first(opCtx0.metasPerNode.get(opCtx0.opNodeId)); - File binDir = binaryWorkDir(snpMgr.snapshotLocalDir(opCtx0.snpName).getAbsolutePath(), - meta.folderName()); + File binDir = binaryWorkDir(snpDir.getAbsolutePath(), meta.folderName()); ctx.cacheObjects().updateMetadata(binDir, opCtx0.stopChecker); } @@ -872,6 +880,7 @@ private IgniteInternalFuture preload(UUID reqId) { Map> allParts = new HashMap<>(); Map> rmtLoadParts = new HashMap<>(); ClusterNode locNode = ctx.cache().context().localNode(); + //ctx.localNodeId().toString().endsWith("2") ? Collections.emptyList() : List locMetas = opCtx0.metasPerNode.get(locNode.id()); // First preload everything from the local node. @@ -894,7 +903,14 @@ private IgniteInternalFuture preload(UUID reqId) { availParts.addAll(parts); } + if (ctx.localNodeId().toString().endsWith("2")) + System.out.println(">xxx> Cache = " + cacheOrGrpName + ", availParts =" + availParts); + List> assignment = affCache.get(cacheOrGrpName).idealAssignment().assignment(); + +// if (ctx.localNodeId().toString().endsWith("2")) +// System.out.println(">xxx> assignment " + assignment); + Set partFuts = availParts .stream() .filter(p -> p != INDEX_PARTITION && assignment.get(p).contains(locNode)) @@ -903,6 +919,9 @@ private IgniteInternalFuture preload(UUID reqId) { allParts.put(grpId, partFuts); + if (ctx.localNodeId().toString().endsWith("2")) + System.out.println(">xxx> partFuts =" + partFuts); + rmtLoadParts.put(grpId, leftParts = new HashSet<>(partFuts)); if (leftParts.isEmpty()) @@ -916,9 +935,6 @@ private IgniteInternalFuture preload(UUID reqId) { if (leftParts.isEmpty()) break; - File snpDir = opCtx0.snpPath != null ? new File(opCtx0.snpPath, opCtx0.snpName) : - ctx.cache().context().snapshotMgr().snapshotLocalDir(opCtx0.snpName); - File snpCacheDir = new File(snpDir, Paths.get(databaseRelativePath(meta.folderName()), dir.getName()).toString()); @@ -965,6 +981,8 @@ private IgniteInternalFuture preload(UUID reqId) { } } + System.out.println(">xxx> rmtLoadParts " + rmtLoadParts); + // Load other partitions from remote nodes. List rmtAwaitParts = rmtLoadParts.values().stream() .flatMap(Collection::stream) @@ -996,6 +1014,8 @@ private IgniteInternalFuture preload(UUID reqId) { ctx.cache().context().snapshotMgr() .requestRemoteSnapshotFiles(m.getKey(), opCtx0.snpName, + // todo + opCtx0.snpPath == null ? null : opCtx0.snpPath.toString(), m.getValue(), opCtx0.stopChecker, (snpFile, t) -> { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index 8f09b0e7d115c7..6a7d35dc82415a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -56,8 +56,10 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2; import org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType; import org.apache.ignite.internal.util.distributed.SingleNodeMessage; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteFuture; @@ -78,6 +80,7 @@ import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PRELOAD; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_START; +import static org.apache.ignite.testframework.GridTestUtils.assertContains; import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; import static org.apache.ignite.testframework.GridTestUtils.runAsync; @@ -126,15 +129,21 @@ public void testClusterSnapshotRestoreWithExplicitPath() throws Exception { try { IgniteEx ignite = null; - for (int i = 0; i < 2; i++) - ignite = startGrid(i); +// dfltCacheCfg.setBackups(0); + dfltCacheCfg.setCacheMode(CacheMode.REPLICATED); + dfltCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 8)); + + for (int i = 0; i < 2; i++) { // IgniteConfiguration cfg = optimize(getConfiguration(getTestIgniteInstanceName(i))); -// + +// cfg.setCacheConfiguration((CacheConfiguration[])null); +// ignite = startGrid(i); + // if (cfgPath) // cfg.setSnapshotPath(snpDir.getAbsolutePath()); -// -// ignite = startGrid(i); -// } + + ignite = startGrid(i); + } // ignite.cluster().baselineAutoAdjustEnabled(false); ignite.cluster().state(ACTIVE); @@ -142,23 +151,36 @@ public void testClusterSnapshotRestoreWithExplicitPath() throws Exception { for (int i = 0; i < CACHE_KEYS_RANGE; i++) ignite.cache(DEFAULT_CACHE_NAME).put(i, i); - IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); +// IgniteSnapshotManager snpMgr = ; - snpMgr.createSnapshot(SNAPSHOT_NAME, snpDir).get(); + ignite.context().cache().context().snapshotMgr().createSnapshot(SNAPSHOT_NAME, snpDir).get(TIMEOUT); -// snpMgr.checkSnapshot(SNAPSHOT_NAME).get(); +// IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, snpDir, null, false).get(TIMEOUT); +// +// StringBuilder sb = new StringBuilder(); +// res.print(sb::append, true); +// +// assertTrue(F.isEmpty(res.exceptions())); +// assertPartitionsSame(res); +// assertContains(log, sb.toString(), "The check procedure has finished, no conflicts have been found"); - ignite.destroyCache(DEFAULT_CACHE_NAME); + ignite = startGrid(2); + ignite.cluster().state(ACTIVE); + resetBaselineTopology(); awaitPartitionMapExchange(); - snpMgr.restoreSnapshot(SNAPSHOT_NAME, null, snpDir).get(); + + ignite.destroyCache(DEFAULT_CACHE_NAME); + awaitPartitionMapExchange(); + + ignite.context().cache().context().snapshotMgr().restoreSnapshot(SNAPSHOT_NAME, null, snpDir).get(); // stopAllGrids(); // IgniteEx snp = startGridsFromSnapshot(2, cfg -> snpDir.getAbsolutePath(), SNAPSHOT_NAME, true); - IgniteCache cache = ignite.cache(SNAPSHOT_NAME); + IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); assert cache != null; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRemoteRequestTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRemoteRequestTest.java index 8d788360ae441c..0ef0740743c046 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRemoteRequestTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRemoteRequestTest.java @@ -79,10 +79,11 @@ public void testSnapshotRemoteRequestFromSingleNode() throws Exception { for (Map.Entry> e : parts.entrySet()) parts0.computeIfAbsent(e.getKey(), k -> new HashSet<>()).addAll(e.getValue()); - IgniteInternalFuture locFut = null; + IgniteInternalFuture locFut; compFut.add(locFut = snp(ignite).requestRemoteSnapshotFiles(grid(1).localNode().id(), SNAPSHOT_NAME, + null, parts, () -> false, defaultPartitionConsumer(parts0, latch))); @@ -123,9 +124,9 @@ public void testSnapshotRemoteRequestEachOther() throws Exception { fromNode0.values().stream().mapToInt(Set::size).sum()); // Snapshot must be taken on node1 and transmitted to node0. - IgniteInternalFuture futFrom1To0 = mgr0.requestRemoteSnapshotFiles(node1, SNAPSHOT_NAME, fromNode1, () -> false, + IgniteInternalFuture futFrom1To0 = mgr0.requestRemoteSnapshotFiles(node1, SNAPSHOT_NAME, null, fromNode1, () -> false, defaultPartitionConsumer(fromNode1, latch)); - IgniteInternalFuture futFrom0To1 = mgr1.requestRemoteSnapshotFiles(node0, SNAPSHOT_NAME, fromNode0, () -> false, + IgniteInternalFuture futFrom0To1 = mgr1.requestRemoteSnapshotFiles(node0, SNAPSHOT_NAME, null, fromNode0, () -> false, defaultPartitionConsumer(fromNode0, latch)); G.allGrids().forEach(g -> TestRecordingCommunicationSpi.spi(g).stopBlock()); @@ -174,6 +175,7 @@ public void testRemoteRequestedInitiatorNodeLeft() throws Exception { snp(ignite).requestRemoteSnapshotFiles(grid(1).localNode().id(), SNAPSHOT_NAME, + null, parts, () -> false, (part, t) -> { @@ -215,6 +217,7 @@ public void testSnapshotRequestRemoteSourceNodeLeft() throws Exception { IgniteInternalFuture fut = snp(ignite).requestRemoteSnapshotFiles(grid(1).localNode().id(), SNAPSHOT_NAME, + null, parts, () -> false, (part, t) -> { @@ -261,6 +264,7 @@ public void testSnapshotRequestRemoteCancel() throws Exception { IgniteInternalFuture fut = snp(ignite).requestRemoteSnapshotFiles(grid(1).localNode().id(), SNAPSHOT_NAME, + null, parts, stopChecker::get, (part, t) -> { From 62d3ec34d804e07da539a72c965f715cebe47e01 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Fri, 27 May 2022 14:35:57 +0300 Subject: [PATCH 05/19] IGNITE-15067 Code cleanup. --- .../AbstractSnapshotVerificationTask.java | 2 +- .../snapshot/IgniteSnapshotManager.java | 170 ++++++++---------- .../snapshot/SnapshotHandlerContext.java | 16 +- .../snapshot/SnapshotHandlerRestoreTask.java | 15 +- .../SnapshotMetadataCollectorTaskArg.java | 16 +- .../snapshot/SnapshotOperationRequest.java | 12 +- .../SnapshotPartitionsVerifyHandler.java | 13 +- .../SnapshotPartitionsVerifyTask.java | 14 +- .../SnapshotPartitionsVerifyTaskArg.java | 14 +- .../SnapshotResponseRemoteFutureTask.java | 13 +- .../snapshot/SnapshotRestoreProcess.java | 8 +- .../snapshot/AbstractSnapshotSelfTest.java | 2 +- .../IgniteClusterSnapshotCheckTest.java | 2 +- .../IgniteClusterSnapshotHandlerTest.java | 5 +- .../IgniteClusterSnapshotRestoreSelfTest.java | 6 +- .../IgniteClusterSnapshotSelfTest.java | 9 +- 16 files changed, 147 insertions(+), 170 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java index 911ecf0620d201..62acaf0a7f4d16 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java @@ -129,5 +129,5 @@ public static void checkMissedMetadata(Collection clusterMetas * @param groups Cache groups to be restored from the snapshot. May be empty if all cache groups are being restored. * @return Compute job. */ - protected abstract ComputeJob createJob(String name, @Nullable File snpPath, String constId, Collection groups); + protected abstract ComputeJob createJob(String name, @Nullable String snpPath, String constId, Collection groups); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 247b15b7443807..181d4a3e7f5bfc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -65,9 +65,9 @@ import java.util.function.BiFunction; import java.util.function.BooleanSupplier; import java.util.function.Consumer; -import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; +import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteInterruptedException; @@ -354,7 +354,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter private volatile ReadWriteMetastorage metaStorage; /** Local snapshot sender factory. */ - private BiFunction locSndrFactory = LocalSnapshotSender::new; + private BiFunction locSndrFactory = LocalSnapshotSender::new; /** Remote snapshot sender factory. */ private BiFunction rmtSndrFactory = this::remoteSnapshotSenderFactory; @@ -577,7 +577,7 @@ public static String partDeltaFileName(int partId) { SNAPSHOT_SYS_VIEW, SNAPSHOT_SYS_VIEW_DESC, new SnapshotViewWalker(), - () -> F.flatCollections(F.transform(localSnapshotNames(), this::readSnapshotMetadatas)), + () -> F.flatCollections(F.transform(localSnapshotNames(), name -> readSnapshotMetadatas(name, null))), this::snapshotViewSupplier); } @@ -689,24 +689,24 @@ public void deleteSnapshot(File snpDir, String folderName) { } } - public File snapshotLocalDir(SnapshotOperationRequest req) { - assert locSnpDir != null; - assert U.alphanumericUnderscore(req.snapshotName()) : req.snapshotName(); - - File locSnpDir0 = req.snapshotPath(); - - return new File(locSnpDir0 == null ? locSnpDir : locSnpDir0, req.snapshotName()); + /** + * @param snpName Snapshot name. + * @return Local snapshot directory for snapshot with given name. + */ + public File snapshotLocalDir(String snpName) { + return snapshotLocalDir(snpName, null); } /** * @param snpName Snapshot name. + * @param snpLocation Snapshot directory path. * @return Local snapshot directory for snapshot with given name. */ - public File snapshotLocalDir(String snpName) { + public File snapshotLocalDir(String snpName, @Nullable String snpLocation) { assert locSnpDir != null; assert U.alphanumericUnderscore(snpName) : snpName; - return new File(locSnpDir, snpName); + return snpLocation == null ? new File(locSnpDir, snpName) : new File(snpLocation, snpName); } /** @@ -714,8 +714,8 @@ public File snapshotLocalDir(String snpName) { * @return Local snapshot directory for snapshot with given name. * @throws IgniteCheckedException If directory doesn't exist. */ - private File resolveSnapshotDir(String snpName, @Nullable File snpPath) throws IgniteCheckedException { - File snpDir = snpPath == null ? snapshotLocalDir(snpName) : snpPath; + private File resolveSnapshotDir(String snpName, @Nullable String snpPath) throws IgniteCheckedException { + File snpDir = snapshotLocalDir(snpName, snpPath); if (!snpDir.exists()) throw new IgniteCheckedException("Snapshot directory doesn't exists: " + snpDir.getAbsolutePath()); @@ -817,14 +817,14 @@ private IgniteInternalFuture initLocalSnapshotStartSt .map(n -> cctx.discovery().node(n).consistentId().toString()) .collect(Collectors.toSet()); - File snpPath = snapshotLocalDir(req); + File snpDir = snapshotLocalDir(req.snapshotName(), req.snapshotPath()); - File smf = new File(snpPath, snapshotMetaFileName(cctx.localNode().consistentId().toString())); + File smf = new File(snpDir, snapshotMetaFileName(cctx.localNode().consistentId().toString())); if (smf.exists()) throw new GridClosureException(new IgniteException("Snapshot metafile must not exist: " + smf.getAbsolutePath())); - snpPath.mkdirs(); + snpDir.mkdirs(); SnapshotMetadata meta = new SnapshotMetadata(req.requestId(), req.snapshotName(), @@ -843,8 +843,7 @@ private IgniteInternalFuture initLocalSnapshotStartSt log.info("Snapshot metafile has been created: " + smf.getAbsolutePath()); } - SnapshotHandlerContext ctx = - new SnapshotHandlerContext(meta, req.groups(), cctx.localNode(), snpPath.getParentFile()); + SnapshotHandlerContext ctx = new SnapshotHandlerContext(meta, req.groups(), cctx.localNode(), snpDir); return new SnapshotOperationResponse(handlers.invokeAll(SnapshotHandlerType.CREATE, ctx)); } @@ -978,7 +977,7 @@ private IgniteInternalFuture initLocalSnapshotEndStag try { if (req.error() != null) - deleteSnapshot(snapshotLocalDir(req), pdsSettings.folderName()); + deleteSnapshot(snapshotLocalDir(req.snapshotName(), req.snapshotPath()), pdsSettings.folderName()); removeLastMetaStorageKey(); } @@ -1207,6 +1206,7 @@ public IgniteInternalFuture checkSnapshot(String name) { * exception is not related to the check procedure, and will be completed normally with the {@code IdleVerifyResult}. * * @param name Snapshot name. + * @param snpPath Snapshot directory path. * @param grps Collection of cache group names to check. * @param includeCustomHandlers {@code True} to invoke all user-defined {@link SnapshotHandlerType#RESTORE} * handlers, otherwise only system consistency check will be performed. @@ -1215,7 +1215,7 @@ public IgniteInternalFuture checkSnapshot(String name) { */ public IgniteInternalFuture checkSnapshot( String name, - @Nullable File snpPath, + @Nullable String snpPath, @Nullable Collection grps, boolean includeCustomHandlers ) { @@ -1319,27 +1319,14 @@ else if (f1.error() instanceof IgniteSnapshotVerifyException) return res; } - /** - * @param snpName Snapshot name. - * @param folderName The name of a directory for the cache group. - * @return The list of cache or cache group names in given snapshot on local node. - */ - public List snapshotCacheDirectories(String snpName, String folderName) { - return snapshotCacheDirectories(snpName, null, folderName, name -> true); - } - -// public List snapshotCacheDirectories(String snpName, @Nullable File snpPath, String folderName) { -// return snapshotCacheDirectories(snpName, snpPath, folderName, name -> true); -// } - /** * @param snpName Snapshot name. * @param folderName The name of a directory for the cache group. * @param names Cache group names to filter. * @return The list of cache or cache group names in given snapshot on local node. */ - public List snapshotCacheDirectories(String snpName, @Nullable File snpPath, String folderName, Predicate names) { - File snpDir = snpPath == null ? snapshotLocalDir(snpName) : new File(snpPath, snpName); + public List snapshotCacheDirectories(String snpName, @Nullable String snpPath, String folderName, Predicate names) { + File snpDir = snapshotLocalDir(snpName, snpPath); if (!snpDir.exists()) return Collections.emptyList(); @@ -1348,16 +1335,12 @@ public List snapshotCacheDirectories(String snpName, @Nullable File snpPat } /** - * @param snpName Snapshot name. + * @param snpDir The full path to the snapshot files. * @param consId Node consistent id to read metadata for. * @return Snapshot metadata instance. */ - public SnapshotMetadata readSnapshotMetadata(String snpName, String consId) { - return readSnapshotMetadata(snpName, null, consId); - } - - public SnapshotMetadata readSnapshotMetadata(String snpName, @Nullable File snpDir, String consId) { - return readSnapshotMetadata(new File(snpDir == null ? snapshotLocalDir(snpName) : snpDir, snapshotMetaFileName(consId))); + public SnapshotMetadata readSnapshotMetadata(File snpDir, String consId) { + return readSnapshotMetadata(new File(snpDir, snapshotMetaFileName(consId))); } /** @@ -1389,19 +1372,16 @@ private SnapshotMetadata readSnapshotMetadata(File smf) { /** * @param snpName Snapshot name. + * @param snpLocation Snapshot directory path. * @return List of snapshot metadata for the given snapshot name on local node. * If snapshot has been taken from local node the snapshot metadata for given * local node will be placed on the first place. */ - public List readSnapshotMetadatas(String snpName) { - return readSnapshotMetadatas(snpName, null); - } - - public List readSnapshotMetadatas(String snpName, @Nullable File snpPath) { + public List readSnapshotMetadatas(String snpName, @Nullable String snpLocation) { A.notNullOrEmpty(snpName, "Snapshot name cannot be null or empty."); A.ensure(U.alphanumericUnderscore(snpName), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_"); - File snpDir = snpPath == null ? snapshotLocalDir(snpName) : new File(snpPath, snpName); + File snpDir = snapshotLocalDir(snpName, snpLocation); if (!(snpDir.exists() && snpDir.isDirectory())) return Collections.emptyList(); @@ -1456,11 +1436,13 @@ public List readSnapshotMetadatas(String snpName, @Nullable Fi } /** - * @param name - * @param snpPath - * @return + * Create a consistent copy of all persistence cache groups from the whole cluster. + * + * @param name Snapshot unique name which satisfies the following name pattern [a-zA-Z0-9_]. + * @param snpPath Snapshot directory path. + * @return Future which will be completed when a process ends. */ - public IgniteFuture createSnapshot(String name, @Nullable File snpPath) { + public IgniteFuture createSnapshot(String name, @Nullable String snpPath) { A.notNullOrEmpty(name, "Snapshot name cannot be null or empty."); A.ensure(U.alphanumericUnderscore(name), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_"); @@ -1544,14 +1526,11 @@ public IgniteFuture createSnapshot(String name, @Nullable File snpPath) { recordSnapshotEvent(name, SNAPSHOT_FAILED_MSG + f.error().getMessage(), EVT_CLUSTER_SNAPSHOT_FAILED); }); - startSnpProc.start(snpFut0.rqId, new SnapshotOperationRequest(snpFut0.rqId, - cctx.localNodeId(), - name, - grps, - new HashSet<>(F.viewReadOnly(srvNodes, - F.node2id(), - (node) -> CU.baselineNode(node, clusterState))), - snpPath)); + Set bltNodeIds = + new HashSet<>(F.viewReadOnly(srvNodes, F.node2id(), (node) -> CU.baselineNode(node, clusterState))); + + startSnpProc.start(snpFut0.rqId, + new SnapshotOperationRequest(snpFut0.rqId, cctx.localNodeId(), name, snpPath, grps, bltNodeIds)); String msg = "Cluster-wide snapshot operation started [snpName=" + name + ", grps=" + grps + ']'; @@ -1575,15 +1554,23 @@ public IgniteFuture createSnapshot(String name, @Nullable File snpPath) { /** {@inheritDoc} */ @Override public IgniteFuture restoreSnapshot(String name, @Nullable Collection grpNames) { - return restoreSnapshot(name, grpNames, null); + return restoreSnapshot(name, null, grpNames); } - public IgniteFuture restoreSnapshot(String name, @Nullable Collection grpNames, @Nullable File snpPath) { + /** + * Restore cache group(s) from the snapshot. + * + * @param name Snapshot name. + * @param snpPath Snapshot directory path. + * @param grpNames Cache groups to be restored or {@code null} to restore all cache groups from the snapshot. + * @return Future which will be completed when restore operation finished. + */ + public IgniteFuture restoreSnapshot(String name, @Nullable String snpPath, @Nullable Collection grpNames) { A.notNullOrEmpty(name, "Snapshot name cannot be null or empty."); A.ensure(U.alphanumericUnderscore(name), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_"); A.ensure(grpNames == null || !grpNames.isEmpty(), "List of cache group names cannot be empty."); - return restoreCacheGrpProc.start(name, grpNames, snpPath); + return restoreCacheGrpProc.start(name, snpPath, grpNames); } /** {@inheritDoc} */ @@ -1606,20 +1593,21 @@ public IgniteFuture restoreSnapshot(String name, @Nullable Collection partitionRowIterator(String snpName, ); } - File snpPart = getPartitionFile(new File(snapshotLocalDir(snpName), databaseRelativePath(folderName)), + File snpPart = getPartitionFile(new File(snapshotLocalDir(snpName, null), databaseRelativePath(folderName)), grps.get(0).getName(), partId); int grpId = CU.cacheId(grpName); @@ -1917,14 +1903,14 @@ private SnapshotFutureTask currentSnapshotTask() { /** * @param factory Factory which produces {@link LocalSnapshotSender} implementation. */ - void localSnapshotSenderFactory(BiFunction factory) { + void localSnapshotSenderFactory(BiFunction factory) { locSndrFactory = factory; } /** * @return Factory which produces {@link LocalSnapshotSender} implementation. */ - BiFunction localSnapshotSenderFactory() { + BiFunction localSnapshotSenderFactory() { return locSndrFactory; } @@ -2116,8 +2102,8 @@ private IgniteFuture executeRestoreManagementTask( * @return Snapshot view. */ private SnapshotView snapshotViewSupplier(SnapshotMetadata meta) { - Collection cacheGrps = F.viewReadOnly(snapshotCacheDirectories(meta.snapshotName(), meta.folderName()), - FilePageStoreManager::cacheGroupName); + List dirs = snapshotCacheDirectories(meta.snapshotName(), meta.folderName(), null, name -> true); + Collection cacheGrps = F.viewReadOnly(dirs, FilePageStoreManager::cacheGroupName); return new SnapshotView(meta.snapshotName(), meta.consistentId(), F.concat(meta.baselineNodes(), ","), F.concat(cacheGrps, ",")); } @@ -2503,7 +2489,7 @@ private static class RemoteSnapshotFilesRecevier extends GridFutureAdapter * @param snpMgr Ignite snapshot manager. * @param rmtNodeId Remote node to request snapshot from. * @param snpName Snapshot name to request. - * @param rmptSnpPath Snapshot directory path. + * @param rmtSnpPath Snapshot directory path. * @param parts Cache group and partitions to request. * @param stopChecker Process interrupt checker. * @param partHnd Partition handler. @@ -2752,14 +2738,11 @@ private Set activeTasks() { } } - File snpDir = reqMsg0.snapshotPath() == null ? - cctx.snapshotMgr().snapshotLocalDir(snpName) : new File(reqMsg0.snapshotPath(), snpName); - AbstractSnapshotFutureTask task = registerTask(rqId, new SnapshotResponseRemoteFutureTask(cctx, nodeId, snpName, - snpDir, + reqMsg0.snapshotPath(), tmpWorkDir, ioFactory, rmtSndrFactory.apply(rqId, nodeId), @@ -3121,9 +3104,6 @@ private Map transmissionParams(String rqId, String cacheDi * Snapshot sender which writes all data to local directory. */ private class LocalSnapshotSender extends SnapshotSender { - /** Snapshot name. */ - private final String snpName; - /** Local snapshot directory. */ private final File snpLocDir; @@ -3134,15 +3114,13 @@ private class LocalSnapshotSender extends SnapshotSender { private final int pageSize; /** - * @param snpPath Snapshot name. * @param snpName Snapshot name. + * @param snpPath Snapshot directory path. */ - public LocalSnapshotSender(String snpName, @Nullable File snpPath) { + public LocalSnapshotSender(String snpName, @Nullable String snpPath) { super(IgniteSnapshotManager.this.log, cctx.kernalContext().pools().getSnapshotExecutorService()); - this.snpName = snpName; - - snpLocDir = snpPath == null ? snapshotLocalDir(snpName) : new File(snpPath, snpName); + snpLocDir = snapshotLocalDir(snpName, snpPath); pageSize = cctx.kernalContext().config().getDataStorageConfiguration().getPageSize(); } @@ -3152,7 +3130,7 @@ public LocalSnapshotSender(String snpName, @Nullable File snpPath) { if (dbDir.exists()) { throw new IgniteException("Snapshot with given name already exists " + - "[snpName=" + snpName + ", absPath=" + dbDir.getAbsolutePath() + ']'); + "[snpName=" + snpLocDir.getName() + ", absPath=" + dbDir.getAbsolutePath() + ']'); } cctx.database().checkpointReadLock(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerContext.java index 73612aae44f18f..ffe18ec3c4efd4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerContext.java @@ -29,8 +29,8 @@ public class SnapshotHandlerContext { /** Snapshot metadata. */ private final SnapshotMetadata metadata; - /** Snapshot directory path. */ - private final String snpBasePath; + /** The full path to the snapshot files. */ + private final File snpDir; /** The names of the cache groups on which the operation is performed. */ private final Collection grps; @@ -42,13 +42,13 @@ public class SnapshotHandlerContext { * @param metadata Snapshot metadata. * @param grps The names of the cache groups on which the operation is performed. * @param locNode Local node. - * @param snpBasePath Snapshot directory path. + * @param snpDir The full path to the snapshot files. */ - public SnapshotHandlerContext(SnapshotMetadata metadata, @Nullable Collection grps, ClusterNode locNode, File snpBasePath) { + public SnapshotHandlerContext(SnapshotMetadata metadata, @Nullable Collection grps, ClusterNode locNode, File snpDir) { this.metadata = metadata; this.grps = grps; this.locNode = locNode; - this.snpBasePath = snpBasePath.toString(); + this.snpDir = snpDir; } /** @@ -59,10 +59,10 @@ public SnapshotMetadata metadata() { } /** - * @return Snapshot directory path. + * @return The full path to the snapshot files. */ - public File snapshotPath() { - return new File(snpBasePath); + public File snapshotDirectory() { + return snpDir; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java index 1e5f376d1fefb2..94e9bf536d695c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java @@ -50,7 +50,7 @@ public class SnapshotHandlerRestoreTask extends AbstractSnapshotVerificationTask /** {@inheritDoc} */ @Override protected ComputeJob createJob( String snpName, - @Nullable File snpPath, + @Nullable String snpPath, String constId, Collection groups ) { @@ -118,7 +118,7 @@ private static class SnapshotHandlerRestoreJob extends ComputeJobAdapter { /** Cache group names. */ private final Collection grps; - private final File snpPath; + private final String snpPath; /** * @param snpName Snapshot name. @@ -126,7 +126,7 @@ private static class SnapshotHandlerRestoreJob extends ComputeJobAdapter { * @param consistentId String representation of the consistent node ID. * @param grps Cache group names. */ - public SnapshotHandlerRestoreJob(String snpName, @Nullable File snpPath, String consistentId, Collection grps) { + public SnapshotHandlerRestoreJob(String snpName, @Nullable String snpPath, String consistentId, Collection grps) { this.snpName = snpName; this.consistentId = consistentId; this.grps = grps; @@ -137,12 +137,11 @@ public SnapshotHandlerRestoreJob(String snpName, @Nullable File snpPath, String @Override public Map> execute() { try { IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); - File snpDir = snpPath == null ? snpMgr.snapshotLocalDir(snpName) : new File(snpPath, snpName); - SnapshotMetadata meta = snpMgr.readSnapshotMetadata(snpName, snpDir, consistentId); + File snpDir = snpMgr.snapshotLocalDir(snpName, snpPath); + SnapshotMetadata meta = snpMgr.readSnapshotMetadata(snpDir, consistentId); - SnapshotHandlerContext ctx = new SnapshotHandlerContext(meta, grps, ignite.localNode(), snpDir.getParentFile()); - - return snpMgr.handlers().invokeAll(SnapshotHandlerType.RESTORE, ctx); + return snpMgr.handlers().invokeAll(SnapshotHandlerType.RESTORE, + new SnapshotHandlerContext(meta, grps, ignite.localNode(), snpDir)); } catch (IgniteCheckedException e) { throw new IgniteException(e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java index d264ed37a1fd69..70d416428fd6f1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTaskArg.java @@ -16,13 +16,13 @@ */ package org.apache.ignite.internal.processors.cache.persistence.snapshot; -import java.io.File; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorDataTransferObject; +import org.jetbrains.annotations.Nullable; /** * Input parameters for checking snapshot partitions consistency task. @@ -35,7 +35,7 @@ public class SnapshotMetadataCollectorTaskArg extends VisorDataTransferObject { private String snpName; /** Snapshot directory path. */ - private File snpPath; + private String snpPath; /** Default constructor. */ public SnapshotMetadataCollectorTaskArg() { @@ -46,7 +46,7 @@ public SnapshotMetadataCollectorTaskArg() { * @param snpName Snapshot name. * @param snpPath Snapshot directory path. */ - public SnapshotMetadataCollectorTaskArg(String snpName, File snpPath) { + public SnapshotMetadataCollectorTaskArg(String snpName, @Nullable String snpPath) { this.snpName = snpName; this.snpPath = snpPath; } @@ -61,24 +61,20 @@ public String snapshotName() { /** * @return Snapshot directory path. */ - public File snapshotPath() { + public String snapshotPath() { return snpPath; } /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeString(out, snpName); - U.writeString(out, snpPath == null ? null : snpPath.toString()); + U.writeString(out, snpPath); } /** {@inheritDoc} */ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { snpName = U.readString(in); - - String path = U.readString(in); - - if (path != null) - snpPath = new File(path); + snpPath = U.readString(in); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java index 5c86f59c8f7cce..08bf1c838e00b9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java @@ -65,24 +65,24 @@ public class SnapshotOperationRequest implements Serializable { * @param reqId Request ID. * @param opNodeId Operational node ID. * @param snpName Snapshot name. + * @param snpPath Snapshot directory path. * @param grps List of cache group names. * @param nodes Baseline node IDs that must be alive to complete the operation. - * @param snpPath Snapshot path. */ public SnapshotOperationRequest( UUID reqId, UUID opNodeId, String snpName, + String snpPath, @Nullable Collection grps, - Set nodes, - @Nullable File snpPath + Set nodes ) { this.reqId = reqId; this.opNodeId = opNodeId; this.snpName = snpName; this.grps = grps; this.nodes = nodes; - this.snpPath = snpPath == null ? null : snpPath.toString(); + this.snpPath = snpPath; } /** @@ -102,8 +102,8 @@ public String snapshotName() { /** * @return Snapshot path. */ - public File snapshotPath() { - return snpPath == null ? null : new File(snpPath); + public String snapshotPath() { + return snpPath; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java index 2d0ee271a722fe..6c408491d66835 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java @@ -67,10 +67,12 @@ import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.fromOrdinal; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_DATA_FILENAME; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheDirectories; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheGroupName; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cachePartitionFiles; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.partId; import static org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId.getTypeByPartId; +import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.databaseRelativePath; import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.calculatePartitionHash; import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.checkPartitionsPageCrcSum; @@ -98,6 +100,9 @@ public SnapshotPartitionsVerifyHandler(GridCacheSharedContext cctx) { /** {@inheritDoc} */ @Override public Map invoke(SnapshotHandlerContext opCtx) throws IgniteCheckedException { + if (!opCtx.snapshotDirectory().exists()) + throw new IgniteCheckedException("Snapshot directory doesn't exists: " + opCtx.snapshotDirectory());; + SnapshotMetadata meta = opCtx.metadata(); Set grps = F.isEmpty(opCtx.groups()) ? new HashSet<>(meta.partitions().keySet()) : @@ -107,9 +112,7 @@ public SnapshotPartitionsVerifyHandler(GridCacheSharedContext cctx) { Map grpDirs = new HashMap<>(); - IgniteSnapshotManager snpMgr = cctx.snapshotMgr(); - - for (File dir : snpMgr.snapshotCacheDirectories(meta.snapshotName(), opCtx.snapshotPath(), meta.folderName(), name -> true)) { + for (File dir : cacheDirectories(new File(opCtx.snapshotDirectory(), databaseRelativePath(meta.folderName())), name -> true)) { int grpId = CU.cacheId(cacheGroupName(dir)); if (!grps.remove(grpId)) @@ -146,7 +149,9 @@ public SnapshotPartitionsVerifyHandler(GridCacheSharedContext cctx) { ThreadLocal buff = ThreadLocal.withInitial(() -> ByteBuffer.allocateDirect(meta.pageSize()) .order(ByteOrder.nativeOrder())); - GridKernalContext snpCtx = snpMgr.createStandaloneKernalContext(meta.snapshotName(), opCtx.snapshotPath(), meta.folderName()); + IgniteSnapshotManager snpMgr = cctx.snapshotMgr(); + + GridKernalContext snpCtx = snpMgr.createStandaloneKernalContext(opCtx.snapshotDirectory(), meta.folderName()); FilePageStoreManager storeMgr = (FilePageStoreManager)cctx.pageStore(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java index ea6e1ff5514a77..5274d9b5f605a8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java @@ -55,7 +55,7 @@ public class SnapshotPartitionsVerifyTask extends AbstractSnapshotVerificationTa private IgniteEx ignite; /** {@inheritDoc} */ - @Override protected ComputeJob createJob(String name, File snpPath, String constId, Collection groups) { + @Override protected ComputeJob createJob(String name, String snpPath, String constId, Collection groups) { return new VisorVerifySnapshotPartitionsJob(name, snpPath, constId, groups); } @@ -87,7 +87,7 @@ private static class VisorVerifySnapshotPartitionsJob extends ComputeJobAdapter private final Collection rqGrps; /** Snapshot directory path. */ - private final File snpPath; + private final String snpPath; /** * @param snpName Snapshot name to validate. @@ -95,7 +95,7 @@ private static class VisorVerifySnapshotPartitionsJob extends ComputeJobAdapter * @param rqGrps Set of cache groups to be checked in the snapshot or {@code empty} to check everything. * @param snpPath Snapshot directory path. */ - public VisorVerifySnapshotPartitionsJob(String snpName, @Nullable File snpPath, String consId, Collection rqGrps) { + public VisorVerifySnapshotPartitionsJob(String snpName, @Nullable String snpPath, String consId, Collection rqGrps) { this.snpName = snpName; this.consId = consId; this.rqGrps = rqGrps; @@ -112,11 +112,11 @@ public VisorVerifySnapshotPartitionsJob(String snpName, @Nullable File snpPath, } try { - SnapshotMetadata meta = cctx.snapshotMgr().readSnapshotMetadata(snpName, consId); - File snpDir = snpPath == null ? cctx.snapshotMgr().snapshotLocalDir(snpName) : new File(snpPath, snpName); + File snpDir = cctx.snapshotMgr().snapshotLocalDir(snpName, snpPath); + SnapshotMetadata meta = cctx.snapshotMgr().readSnapshotMetadata(snpDir, consId); return new SnapshotPartitionsVerifyHandler(cctx) - .invoke(new SnapshotHandlerContext(meta, rqGrps, ignite.localNode(), snpDir.getParentFile())); + .invoke(new SnapshotHandlerContext(meta, rqGrps, ignite.localNode(), snpDir)); } catch (IgniteCheckedException e) { throw new IgniteException(e); @@ -132,7 +132,7 @@ public VisorVerifySnapshotPartitionsJob(String snpName, @Nullable File snpPath, return false; VisorVerifySnapshotPartitionsJob job = (VisorVerifySnapshotPartitionsJob)o; - + // todo rqGrps snpPath return snpName.equals(job.snpName) && consId.equals(job.consId); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTaskArg.java index 2f54a317603d96..a26c8d0cd1a8d7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTaskArg.java @@ -42,7 +42,7 @@ public class SnapshotPartitionsVerifyTaskArg extends VisorDataTransferObject { /** The map of distribution of snapshot metadata pieces across the cluster. */ private Map> clusterMetas; - private File snpPath; + private String snpPath; /** Default constructor. */ public SnapshotPartitionsVerifyTaskArg() { @@ -57,7 +57,7 @@ public SnapshotPartitionsVerifyTaskArg() { public SnapshotPartitionsVerifyTaskArg( Collection grpNames, Map> clusterMetas, - @Nullable File snpPath + @Nullable String snpPath ) { this.grpNames = grpNames; this.clusterMetas = clusterMetas; @@ -81,7 +81,7 @@ public Map> clusterMetadata() { /** * @return Snapshot directory path. */ - public File snapshotPath() { + public String snapshotPath() { return snpPath; } @@ -89,18 +89,14 @@ public File snapshotPath() { @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeCollection(out, grpNames); U.writeMap(out, clusterMetas); - U.writeString(out, snpPath == null ? null : snpPath.toString()); + U.writeString(out, snpPath); } /** {@inheritDoc} */ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { grpNames = U.readCollection(in); clusterMetas = U.readMap(in); - - String snpPathname = U.readString(in); - - if (snpPathname != null) - snpPath = new File(snpPathname); + snpPath = U.readString(in); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java index d79fb480076559..fa974d4b5666ad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java @@ -40,13 +40,13 @@ /** */ public class SnapshotResponseRemoteFutureTask extends AbstractSnapshotFutureTask { /** */ - private final File snpDir; + private final String snpPath; /** * @param cctx Shared context. * @param srcNodeId Node id which cause snapshot task creation. * @param snpName Unique identifier of snapshot process. - * @param snpDir + * @param snpPath * @param tmpWorkDir Working directory for intermediate snapshot results. * @param ioFactory Factory to working with snapshot files. * @param snpSndr Factory which produces snapshot receiver instance. @@ -55,7 +55,7 @@ public SnapshotResponseRemoteFutureTask( GridCacheSharedContext cctx, UUID srcNodeId, String snpName, - File snpDir, + String snpPath, File tmpWorkDir, FileIOFactory ioFactory, SnapshotSender snpSndr, @@ -63,7 +63,7 @@ public SnapshotResponseRemoteFutureTask( ) { super(cctx, srcNodeId, snpName, tmpWorkDir, ioFactory, snpSndr, parts); - this.snpDir = snpDir; + this.snpPath = snpPath; } /** {@inheritDoc} */ @@ -81,9 +81,10 @@ public SnapshotResponseRemoteFutureTask( snpSndr.init(handled.size()); + File snpDir = cctx.snapshotMgr().snapshotLocalDir(snpName, snpPath); + List> futs = new ArrayList<>(); - // todo - List metas = cctx.snapshotMgr().readSnapshotMetadatas(snpName, snpDir.getParentFile()); + List metas = cctx.snapshotMgr().readSnapshotMetadatas(snpName, snpPath); for (SnapshotMetadata meta : metas) { Map> parts0 = meta.partitions(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 75a1b77c1f42e9..5f2019da643ed4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -208,10 +208,11 @@ protected void registerMetrics() { * Start cache group restore operation. * * @param snpName Snapshot name. + * @param snpPath Snapshot directory path. * @param cacheGrpNames Cache groups to be restored or {@code null} to restore all cache groups from the snapshot. * @return Future that will be completed when the restore operation is complete and the cache groups are started. */ - public IgniteFuture start(String snpName, @Nullable Collection cacheGrpNames, @Nullable File snpPath) { + public IgniteFuture start(String snpName, @Nullable String snpPath, @Nullable Collection cacheGrpNames) { IgniteSnapshotManager snpMgr = ctx.cache().context().snapshotMgr(); ClusterSnapshotFuture fut0; @@ -336,7 +337,7 @@ public IgniteFuture start(String snpName, @Nullable Collection cac Collection bltNodes = F.viewReadOnly(ctx.discovery().discoCache().aliveBaselineNodes(), F.node2id()); SnapshotOperationRequest req = new SnapshotOperationRequest( - fut0.rqId, F.first(dataNodes), snpName, cacheGrpNames, new HashSet<>(bltNodes), snpPath); + fut0.rqId, F.first(dataNodes), snpName, snpPath, cacheGrpNames, new HashSet<>(bltNodes)); prepareRestoreProc.start(req.requestId(), req); }); @@ -1014,7 +1015,6 @@ private IgniteInternalFuture preload(UUID reqId) { ctx.cache().context().snapshotMgr() .requestRemoteSnapshotFiles(m.getKey(), opCtx0.snpName, - // todo opCtx0.snpPath == null ? null : opCtx0.snpPath.toString(), m.getValue(), opCtx0.stopChecker, @@ -1406,7 +1406,7 @@ private static class SnapshotRestoreContext { /** Snapshot name. */ private final String snpName; - private final File snpPath; + private final String snpPath; /** Baseline discovery cache for node IDs that must be alive to complete the operation.*/ private final DiscoCache discoCache; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java index 1212e410b4be63..0e5fc1f091fa72 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java @@ -562,7 +562,7 @@ protected static List setBlockingSnapshotExecutor(List old = mgr.localSnapshotSenderFactory(); + BiFunction old = mgr.localSnapshotSenderFactory(); BlockingExecutor block = new BlockingExecutor(mgr.snapshotExecutorService()); execs.add(block); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java index a3106d97205b69..bce14dac639b57 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java @@ -477,7 +477,7 @@ public void testClusterSnapshotCheckHashesSameAsIdleVerifyHashes() throws Except IdleVerifyResultV2 snpVerifyRes = ignite.compute().execute(new TestSnapshotPartitionsVerifyTask(), new SnapshotPartitionsVerifyTaskArg(new HashSet<>(), Collections.singletonMap(ignite.cluster().localNode(), - Collections.singletonList(snp(ignite).readSnapshotMetadata(SNAPSHOT_NAME, + Collections.singletonList(snp(ignite).readSnapshotMetadata(snp(ignite).snapshotLocalDir(SNAPSHOT_NAME), (String)ignite.configuration().getConsistentId()))), null)) .idleVerifyResult(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java index 6f87fd502f9a87..872b6b97b1c804 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java @@ -150,9 +150,10 @@ private void changeMetadataRequestIdOnDisk(UUID newReqId) throws Exception { for (Ignite grid : G.allGrids()) { IgniteSnapshotManager snpMgr = ((IgniteEx)grid).context().cache().context().snapshotMgr(); String constId = grid.cluster().localNode().consistentId().toString(); + File snpDir = snpMgr.snapshotLocalDir(SNAPSHOT_NAME); - SnapshotMetadata metadata = snpMgr.readSnapshotMetadata(SNAPSHOT_NAME, constId); - File smf = new File(snpMgr.snapshotLocalDir(SNAPSHOT_NAME), U.maskForFileName(constId) + SNAPSHOT_METAFILE_EXT); + SnapshotMetadata metadata = snpMgr.readSnapshotMetadata(snpDir, constId); + File smf = new File(snpDir, U.maskForFileName(constId) + SNAPSHOT_METAFILE_EXT); try (OutputStream out = new BufferedOutputStream(new FileOutputStream(smf))) { GridTestUtils.setFieldValue(metadata, "rqId", newReqId); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index 6a7d35dc82415a..a4aaec360e4caa 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -153,7 +153,7 @@ public void testClusterSnapshotRestoreWithExplicitPath() throws Exception { // IgniteSnapshotManager snpMgr = ; - ignite.context().cache().context().snapshotMgr().createSnapshot(SNAPSHOT_NAME, snpDir).get(TIMEOUT); + ignite.context().cache().context().snapshotMgr().createSnapshot(SNAPSHOT_NAME, snpDir.toString()).get(TIMEOUT); // IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, snpDir, null, false).get(TIMEOUT); // @@ -165,7 +165,7 @@ public void testClusterSnapshotRestoreWithExplicitPath() throws Exception { // assertContains(log, sb.toString(), "The check procedure has finished, no conflicts have been found"); ignite = startGrid(2); - ignite.cluster().state(ACTIVE); +// ignite.cluster().state(ACTIVE); resetBaselineTopology(); awaitPartitionMapExchange(); @@ -174,7 +174,7 @@ public void testClusterSnapshotRestoreWithExplicitPath() throws Exception { ignite.destroyCache(DEFAULT_CACHE_NAME); awaitPartitionMapExchange(); - ignite.context().cache().context().snapshotMgr().restoreSnapshot(SNAPSHOT_NAME, null, snpDir).get(); + ignite.context().cache().context().snapshotMgr().restoreSnapshot(SNAPSHOT_NAME, snpDir.toString(), null).get(); // stopAllGrids(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java index e201d58bd73d69..30a87d19ee8f20 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java @@ -743,7 +743,8 @@ private void doTestClusterSnapshotWithExplicitPath(boolean cfgPath) throws Excep for (int i = 0; i < CACHE_KEYS_RANGE; i++) ignite.cache(DEFAULT_CACHE_NAME).put(i, i); - ignite.context().cache().context().snapshotMgr().createSnapshot(SNAPSHOT_NAME, cfgPath ? null : snpDir).get(); + ignite.context().cache().context().snapshotMgr() + .createSnapshot(SNAPSHOT_NAME, cfgPath ? null : snpDir.getAbsolutePath()).get(); stopAllGrids(); @@ -1106,7 +1107,7 @@ public void testConcurrentClusterSnapshotFromClient() throws Exception { IgniteEx clnt = startClientGrid(2); IgniteSnapshotManager mgr = snp(grid); - BiFunction old = mgr.localSnapshotSenderFactory(); + BiFunction old = mgr.localSnapshotSenderFactory(); BlockingExecutor block = new BlockingExecutor(mgr.snapshotExecutorService()); @@ -1172,11 +1173,11 @@ public void testClusterSnapshotFinishedTryCancel() throws Exception { * @param blocked Latch to await delta partition processing. * @return Factory which produces local snapshot senders. */ - private BiFunction blockingLocalSnapshotSender(IgniteEx ignite, + private BiFunction blockingLocalSnapshotSender(IgniteEx ignite, CountDownLatch started, CountDownLatch blocked ) { - BiFunction old = snp(ignite).localSnapshotSenderFactory(); + BiFunction old = snp(ignite).localSnapshotSenderFactory(); return (snpName, snpPath) -> new DelegateSnapshotSender(log, snp(ignite).snapshotExecutorService(), old.apply(snpName, null)) { @Override public void sendDelta0(File delta, String cacheDirName, GroupPartitionId pair) { From 8e396b829978d2d339369103d3fdb54ea381a295 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Mon, 30 May 2022 11:03:32 +0300 Subject: [PATCH 06/19] IGNITE-15067 Add control.sh option. --- docs/_docs/snapshots/snapshots.adoc | 22 ++++++---- .../snapshot/SnapshotCreateCommand.java | 20 ++++++--- .../snapshot/SnapshotCreateCommandOption.java | 24 +++++++---- .../snapshot/SnapshotRestoreCommand.java | 22 +++++++--- .../SnapshotRestoreCommandOption.java | 4 ++ .../ignite/util/GridCommandHandlerTest.java | 43 +++++++++++++++++++ .../snapshot/VisorSnapshotCreateTask.java | 3 +- .../snapshot/VisorSnapshotCreateTaskArg.java | 14 +++++- .../snapshot/VisorSnapshotRestoreTask.java | 4 +- .../snapshot/VisorSnapshotRestoreTaskArg.java | 3 +- ...mmandHandlerClusterByClassTest_help.output | 6 ++- ...ndlerClusterByClassWithSSLTest_help.output | 6 ++- 12 files changed, 136 insertions(+), 35 deletions(-) diff --git a/docs/_docs/snapshots/snapshots.adoc b/docs/_docs/snapshots/snapshots.adoc index 4764fc20f2b416..b3f03cdc766e0f 100644 --- a/docs/_docs/snapshots/snapshots.adoc +++ b/docs/_docs/snapshots/snapshots.adoc @@ -105,17 +105,20 @@ Ignite ships the link:tools/control-script[Control Script] that supports snapsho [source,shell] ---- -#Create a cluster snapshot in the background: -control.(sh|bat) --snapshot create snapshot_name +# Create a cluster snapshot named "snapshot_09062021" in the background: +control.(sh|bat) --snapshot create snapshot_09062021 -#Create a cluster snapshot (wait for the entire operation to complete): -control.(sh|bat) --snapshot create snapshot_name --sync +# Create a cluster snapshot named "snapshot_09062021" and wait for the entire operation to complete: +control.(sh|bat) --snapshot create snapshot_09062021 --sync -#Cancel a running snapshot: -control.(sh|bat) --snapshot cancel snapshot_name +# Create a cluster snapshot named "snapshot_09062021" in the "/tmp/ignite/snapshots" folder (the full path to the snapshot files will be /tmp/ignite/snapshots/snapshot_09062021): +control.(sh|bat) --snapshot create snapshot_09062021 -dest /tmp/ignite/snapshots -#Kill a running snapshot: -control.(sh|bat) --kill SNAPSHOT snapshot_name +# Cancel a running snapshot named "snapshot_09062021": +control.(sh|bat) --snapshot cancel snapshot_09062021 + +# Kill a running snapshot named "snapshot_09062021": +control.(sh|bat) --kill SNAPSHOT snapshot_09062021 ---- === Using JMX @@ -225,6 +228,9 @@ control.(sh|bat) --snapshot restore snapshot_09062021 --start # Start restoring all user-created cache groups from the snapshot "snapshot_09062021" and wait for the entire operation to complete. control.(sh|bat) --snapshot restore snapshot_09062021 --start --sync +# Start restoring all user-created cache groups from the snapshot "snapshot_09062021" located in the "/tmp/ignite/snapshots" folder (the full path to the snapshot files should be /tmp/ignite/snapshots/snapshot_09062021): +control.(sh|bat) --snapshot restore snapshot_09062021 --src /tmp/ignite/snapshots + # Start restoring only "cache-group1" and "cache-group2" from the snapshot "snapshot_09062021" in the background. control.(sh|bat) --snapshot restore snapshot_09062021 --start --groups cache-group1,cache-group2 diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java index 6c2e10a8336d36..97e4d22c83294d 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java @@ -28,6 +28,7 @@ import static org.apache.ignite.internal.commandline.CommandList.SNAPSHOT; import static org.apache.ignite.internal.commandline.CommandLogger.optional; +import static org.apache.ignite.internal.commandline.snapshot.SnapshotCreateCommandOption.DESTINATION; import static org.apache.ignite.internal.commandline.snapshot.SnapshotCreateCommandOption.SYNC; /** @@ -42,6 +43,7 @@ protected SnapshotCreateCommand() { /** {@inheritDoc} */ @Override public void parseArguments(CommandArgIterator argIter) { String snpName = argIter.nextArg("Expected snapshot name."); + String snpPath = null; boolean sync = false; while (argIter.hasNextSubArg()) { @@ -53,22 +55,30 @@ protected SnapshotCreateCommand() { throw new IllegalArgumentException("Invalid argument: " + arg + ". " + "Possible options: " + F.concat(F.asList(SnapshotCreateCommandOption.values()), ", ") + '.'); } + else if (option == DESTINATION) { + String errMsg = "Expected path to snapshot directory."; - // The snapshot create command currently supports only one optional argument. - assert option == SYNC; + if (CommandArgIterator.isCommandOrOption(argIter.peekNextArg())) + throw new IllegalArgumentException(errMsg); + + snpPath = argIter.nextArg(errMsg); + } + else if (option == SYNC) + sync = true; - sync = true; } - cmdArg = new VisorSnapshotCreateTaskArg(snpName, sync); + cmdArg = new VisorSnapshotCreateTaskArg(snpName, snpPath, sync); } /** {@inheritDoc} */ @Override public void printUsage(Logger log) { Map params = new LinkedHashMap<>(generalUsageOptions()); + params.put(DESTINATION.optionName(), DESTINATION.description()); params.put(SYNC.optionName(), SYNC.description()); - usage(log, "Create cluster snapshot:", SNAPSHOT, params, name(), SNAPSHOT_NAME_ARG, optional(SYNC.argName())); + usage(log, "Create cluster snapshot:", SNAPSHOT, params, name(), SNAPSHOT_NAME_ARG, + optional(DESTINATION.argName(), DESTINATION.optionName()), optional(SYNC.argName())); } } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java index 4647a4829cb4d5..9bb173d310d0c5 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java @@ -24,32 +24,40 @@ */ public enum SnapshotCreateCommandOption implements CommandArg { /** Synchronous execution flag. */ - SYNC("sync", "Run the operation synchronously, the command will wait for the entire operation to complete. " + - "Otherwise, it will be performed in the background, and the command will immediately return control."); + SYNC("--sync", "sync", "Run the operation synchronously, the command will wait for the entire operation to complete. " + + "Otherwise, it will be performed in the background, and the command will immediately return control."), + + /** Snapshot directory path. */ + DESTINATION("--dest", "path", "The path to directory where the snapshot will be saved. If not specified, " + + "the default snapshot directory will be used."); + + /** Option name. */ + private final String argName; /** Option name. */ - private final String name; + private final String optionName; /** Option description. */ private final String desc; /** - * @param name Option name. + * @param argName Option name. * @param desc Option description. */ - SnapshotCreateCommandOption(String name, String desc) { - this.name = name; + SnapshotCreateCommandOption(String argName, String optionName, String desc) { + this.argName = argName; + this.optionName = optionName; this.desc = desc; } /** {@inheritDoc} */ @Override public String argName() { - return "--" + name; + return argName; } /** @return Option name. */ public String optionName() { - return name; + return optionName; } /** @return Option description. */ diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java index e154d58357ef64..eff09eb52ae142 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java @@ -33,8 +33,10 @@ import static org.apache.ignite.internal.commandline.CommandList.SNAPSHOT; import static org.apache.ignite.internal.commandline.CommandLogger.optional; import static org.apache.ignite.internal.commandline.snapshot.SnapshotRestoreCommandOption.GROUPS; +import static org.apache.ignite.internal.commandline.snapshot.SnapshotRestoreCommandOption.SOURCE; import static org.apache.ignite.internal.commandline.snapshot.SnapshotRestoreCommandOption.SYNC; import static org.apache.ignite.internal.commandline.snapshot.SnapshotSubcommands.RESTORE; +import static org.apache.ignite.internal.visor.snapshot.VisorSnapshotRestoreTaskAction.START; /** * Sub-command to restore snapshot. @@ -58,13 +60,14 @@ protected SnapshotRestoreCommand() { @Override public void parseArguments(CommandArgIterator argIter) { String snpName = argIter.nextArg("Expected snapshot name."); VisorSnapshotRestoreTaskAction restoreAction = parseAction(argIter); + String snpPath = null; Set grpNames = null; boolean sync = false; while (argIter.hasNextSubArg()) { String arg = argIter.nextArg(null); - if (restoreAction != VisorSnapshotRestoreTaskAction.START) { + if (restoreAction != START) { throw new IllegalArgumentException("Invalid argument: " + arg + ". " + "Action \"--" + restoreAction.name().toLowerCase() + "\" does not support specified option."); } @@ -75,7 +78,7 @@ protected SnapshotRestoreCommand() { throw new IllegalArgumentException("Invalid argument: " + arg + ". " + "Possible options: " + F.concat(F.asList(SnapshotRestoreCommandOption.values()), ", ") + '.'); } - else if (option == SnapshotRestoreCommandOption.GROUPS) { + else if (option == GROUPS) { String argDesc = "a comma-separated list of cache group names."; grpNames = argIter.nextStringSet(argDesc); @@ -83,11 +86,19 @@ else if (option == SnapshotRestoreCommandOption.GROUPS) { if (grpNames.isEmpty()) throw new IllegalArgumentException("Expected " + argDesc); } + else if (option == SOURCE) { + String errMsg = "Expected path to snapshot directory."; + + if (CommandArgIterator.isCommandOrOption(argIter.peekNextArg())) + throw new IllegalArgumentException(errMsg); + + snpPath = argIter.nextArg(errMsg); + } else if (option == SYNC) sync = true; } - cmdArg = new VisorSnapshotRestoreTaskArg(snpName, sync, restoreAction, grpNames); + cmdArg = new VisorSnapshotRestoreTaskArg(snpName, snpPath, sync, restoreAction, grpNames); } /** {@inheritDoc} */ @@ -96,10 +107,11 @@ else if (option == SYNC) Map startParams = new LinkedHashMap<>(params); startParams.put(GROUPS.optionName(), GROUPS.description()); + startParams.put(SOURCE.optionName(), SOURCE.description()); startParams.put(SYNC.optionName(), SYNC.description()); usage(log, "Restore snapshot:", SNAPSHOT, startParams, RESTORE.toString(), SNAPSHOT_NAME_ARG, "--start", - optional(GROUPS.argName(), GROUPS.optionName()), optional(SYNC.argName())); + optional(GROUPS.argName(), GROUPS.optionName()), optional(SOURCE.argName(), SOURCE.optionName()), optional(SYNC.argName())); usage(log, "Snapshot restore operation status:", SNAPSHOT, params, RESTORE.toString(), SNAPSHOT_NAME_ARG, "--status"); usage(log, "Cancel snapshot restore operation:", SNAPSHOT, params, RESTORE.toString(), SNAPSHOT_NAME_ARG, "--cancel"); } @@ -108,7 +120,7 @@ else if (option == SYNC) @Override public String confirmationPrompt() { VisorSnapshotRestoreTaskArg arg = (VisorSnapshotRestoreTaskArg)cmdArg; - return arg.jobAction() != VisorSnapshotRestoreTaskAction.START || arg.groupNames() != null ? null : + return arg.jobAction() != START || arg.groupNames() != null ? null : "Warning: command will restore ALL USER-CREATED CACHE GROUPS from the snapshot " + arg.snapshotName() + '.'; } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java index b558a6be67a88e..3520de7305649c 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java @@ -26,6 +26,10 @@ public enum SnapshotRestoreCommandOption implements CommandArg { /** Cache group names. */ GROUPS("--groups", "group1,...groupN", "Cache group names."), + /** Snapshot directory location. */ + SOURCE("--src", "path", "The path to the directory where the snapshot files are located. If not specified, " + + "the default snapshot directory will be used."), + /** Synchronous execution flag. */ SYNC(SnapshotCreateCommandOption.SYNC.argName(), SnapshotCreateCommandOption.SYNC.optionName(), SnapshotCreateCommandOption.SYNC.description()); diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 1e01c88587e830..9e33cda79f5a89 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3348,6 +3348,49 @@ public void testSnapshotRestore() throws Exception { } } + /** @throws Exception If fails. */ + @Test + public void testSnapshotCreateRestoreCustomDir() throws Exception { + int keysCnt = 100; + + File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); + + assert snpDir.list().length == 0 : "Target directory is not empty: " + Arrays.asList(snpDir.list()); + + String snpName = "snapshot_30052022"; + + try { + Ignite ignite = startGrids(2); + ignite.cluster().state(ACTIVE); + + createCacheAndPreload(ignite, keysCnt); + + injectTestSystemOut(); + + CommandHandler h = new CommandHandler(); + + assertEquals(EXIT_CODE_OK, + execute(h, "--snapshot", "create", snpName, "--sync", "--dest", snpDir.getAbsolutePath())); + + ignite.destroyCache(DEFAULT_CACHE_NAME); + + assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "restore", snpName, "--start", "--sync")); + assertContains(log, testOut.toString(), "Snapshot does not exists [snapshot=" + snpName); + + assertEquals(EXIT_CODE_OK, + execute(h, "--snapshot", "restore", snpName, "--start", "--sync", "--src", snpDir.getAbsolutePath())); + + IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); + + assertEquals(keysCnt, cache.size()); + + for (int i = 0; i < keysCnt; i++) + assertEquals(Integer.valueOf(i), cache.get(i)); + } finally { + U.delete(snpDir); + } + } + /** @throws Exception If fails. */ @Test public void testSnapshotRestoreCancelAndStatus() throws Exception { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTask.java index 45232bc9549842..2879c4aeceb53d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTask.java @@ -51,7 +51,8 @@ protected VisorSnapshotCreateJob(VisorSnapshotCreateTaskArg arg, boolean debug) /** {@inheritDoc} */ @Override protected String run(VisorSnapshotCreateTaskArg arg) throws IgniteException { - IgniteFuture fut = ignite.snapshot().createSnapshot(arg.snapshotName()); + IgniteFuture fut = + ignite.context().cache().context().snapshotMgr().createSnapshot(arg.snapshotName(), arg.snapshotPath()); if (arg.sync() || fut.isDone()) fut.get(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTaskArg.java index 04855257cb664a..e1a9ba00687f1f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTaskArg.java @@ -34,6 +34,9 @@ public class VisorSnapshotCreateTaskArg extends IgniteDataTransferObject { /** Snapshot name. */ private String snpName; + /** Snapshot directory path. */ + private String snpPath; + /** Synchronous execution flag. */ private boolean sync; @@ -44,10 +47,12 @@ public VisorSnapshotCreateTaskArg() { /** * @param snpName Snapshot name. + * @param snpPath Snapshot directory path. * @param sync Synchronous execution flag. */ - public VisorSnapshotCreateTaskArg(String snpName, boolean sync) { + public VisorSnapshotCreateTaskArg(String snpName, String snpPath, boolean sync) { this.snpName = snpName; + this.snpPath = snpPath; this.sync = sync; } @@ -56,6 +61,11 @@ public String snapshotName() { return snpName; } + /** @return Snapshot directory path. */ + public String snapshotPath() { + return snpPath; + } + /** @return Synchronous execution flag. */ public boolean sync() { return sync; @@ -64,12 +74,14 @@ public boolean sync() { /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeString(out, snpName); + U.writeString(out, snpPath); out.writeBoolean(sync); } /** {@inheritDoc} */ @Override protected void readExternalData(byte ver, ObjectInput in) throws IOException, ClassNotFoundException { snpName = U.readString(in); + snpPath = U.readString(in); sync = in.readBoolean(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTask.java index 56d35b5c9e382c..49e5e62d1668fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTask.java @@ -63,8 +63,8 @@ protected VisorSnapshotStartRestoreJob(VisorSnapshotRestoreTaskArg arg, boolean /** {@inheritDoc} */ @Override protected String run(VisorSnapshotRestoreTaskArg arg) throws IgniteException { - IgniteFuture fut = - ignite.context().cache().context().snapshotMgr().restoreSnapshot(arg.snapshotName(), arg.groupNames()); + IgniteFuture fut = ignite.context().cache().context().snapshotMgr() + .restoreSnapshot(arg.snapshotName(), arg.snapshotPath(), arg.groupNames()); if (arg.sync() || fut.isDone()) fut.get(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTaskArg.java index efad1bfc1dee55..86c07c4740a8e4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotRestoreTaskArg.java @@ -51,11 +51,12 @@ public VisorSnapshotRestoreTaskArg() { */ public VisorSnapshotRestoreTaskArg( String snpName, + String snpPath, boolean sync, VisorSnapshotRestoreTaskAction action, @Nullable Collection grpNames ) { - super(snpName, sync); + super(snpName, snpPath, sync); this.action = action; this.grpNames = grpNames; diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output index b70f2955cbe4bb..b18f60f0f3d047 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output @@ -140,10 +140,11 @@ This utility can do the following commands: snapshot_name - Snapshot name. Create cluster snapshot: - control.(sh|bat) --snapshot create snapshot_name [--sync] + control.(sh|bat) --snapshot create snapshot_name [--dest path] [--sync] Parameters: snapshot_name - Snapshot name. + path - The path to directory where the snapshot will be saved. If not specified, the default snapshot directory will be used. sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Cancel running snapshot: @@ -159,11 +160,12 @@ This utility can do the following commands: snapshot_name - Snapshot name. Restore snapshot: - control.(sh|bat) --snapshot restore snapshot_name --start [--groups group1,...groupN] [--sync] + control.(sh|bat) --snapshot restore snapshot_name --start [--groups group1,...groupN] [--src path] [--sync] Parameters: snapshot_name - Snapshot name. group1,...groupN - Cache group names. + path - The path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Snapshot restore operation status: diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output index b70f2955cbe4bb..b18f60f0f3d047 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output @@ -140,10 +140,11 @@ This utility can do the following commands: snapshot_name - Snapshot name. Create cluster snapshot: - control.(sh|bat) --snapshot create snapshot_name [--sync] + control.(sh|bat) --snapshot create snapshot_name [--dest path] [--sync] Parameters: snapshot_name - Snapshot name. + path - The path to directory where the snapshot will be saved. If not specified, the default snapshot directory will be used. sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Cancel running snapshot: @@ -159,11 +160,12 @@ This utility can do the following commands: snapshot_name - Snapshot name. Restore snapshot: - control.(sh|bat) --snapshot restore snapshot_name --start [--groups group1,...groupN] [--sync] + control.(sh|bat) --snapshot restore snapshot_name --start [--groups group1,...groupN] [--src path] [--sync] Parameters: snapshot_name - Snapshot name. group1,...groupN - Cache group names. + path - The path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Snapshot restore operation status: From 8c5711eb4ed2fa0acd4df4c2fc1f9a4c6754c0b0 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Mon, 30 May 2022 18:32:44 +0300 Subject: [PATCH 07/19] IGNITE-15067 Check command + checkstyle fixes. --- .../snapshot/SnapshotCheckCommand.java | 42 +++++++++- .../snapshot/SnapshotCheckCommandOption.java | 68 ++++++++++++++++ .../snapshot/SnapshotCreateCommand.java | 2 +- .../snapshot/SnapshotCreateCommandOption.java | 2 +- .../snapshot/SnapshotRestoreCommand.java | 2 +- .../SnapshotRestoreCommandOption.java | 2 +- .../ignite/util/GridCommandHandlerTest.java | 12 ++- .../AbstractSnapshotVerificationTask.java | 1 - .../snapshot/IgniteSnapshotManager.java | 8 +- .../snapshot/SnapshotHandlerRestoreTask.java | 3 +- .../snapshot/SnapshotOperationRequest.java | 5 +- .../SnapshotPartitionsVerifyTaskArg.java | 4 +- .../snapshot/SnapshotRestoreProcess.java | 22 +---- .../snapshot/VisorSnapshotCheckTask.java | 20 ++--- .../snapshot/VisorSnapshotCheckTaskArg.java | 80 +++++++++++++++++++ .../snapshot/VisorSnapshotCreateTaskArg.java | 32 ++------ .../resources/META-INF/classnames.properties | 1 + .../snapshot/EncryptedSnapshotTest.java | 4 +- .../IgniteClusterSnapshotCheckTest.java | 22 ++--- .../IgniteClusterSnapshotRestoreSelfTest.java | 55 ++++--------- .../IgniteClusterSnapshotSelfTest.java | 1 - .../IgniteSnapshotRestoreFromRemoteTest.java | 2 +- ...mmandHandlerClusterByClassTest_help.output | 7 +- ...ndlerClusterByClassWithSSLTest_help.output | 7 +- ...teClusterSnapshotCheckWithIndexesTest.java | 6 +- 25 files changed, 274 insertions(+), 136 deletions(-) create mode 100644 modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCheckTaskArg.java diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java index 9bbf9ea76f1ea9..b3003eff78b33c 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java @@ -17,11 +17,19 @@ package org.apache.ignite.internal.commandline.snapshot; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.logging.Logger; +import org.apache.ignite.internal.commandline.CommandArgIterator; +import org.apache.ignite.internal.commandline.argument.CommandArgUtils; import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.visor.snapshot.VisorSnapshotCheckTask; +import org.apache.ignite.internal.visor.snapshot.VisorSnapshotCheckTaskArg; import static org.apache.ignite.internal.commandline.CommandList.SNAPSHOT; +import static org.apache.ignite.internal.commandline.CommandLogger.optional; +import static org.apache.ignite.internal.commandline.snapshot.SnapshotCheckCommandOption.SOURCE; /** * Sub-command to check snapshot. @@ -32,9 +40,41 @@ protected SnapshotCheckCommand() { super("check", VisorSnapshotCheckTask.class); } + /** {@inheritDoc} */ + @Override public void parseArguments(CommandArgIterator argIter) { + String snpName = argIter.nextArg("Expected snapshot name."); + String snpPath = null; + + while (argIter.hasNextSubArg()) { + String arg = argIter.nextArg(null); + + SnapshotCheckCommandOption option = CommandArgUtils.of(arg, SnapshotCheckCommandOption.class); + + if (option == null) { + throw new IllegalArgumentException("Invalid argument: " + arg + ". " + + "Possible options: " + F.concat(F.asList(SnapshotCheckCommandOption.values()), ", ") + '.'); + } + else if (option == SOURCE) { + String errMsg = "Expected path to the snapshot directory."; + + if (CommandArgIterator.isCommandOrOption(argIter.peekNextArg())) + throw new IllegalArgumentException(errMsg); + + snpPath = argIter.nextArg(errMsg); + } + } + + cmdArg = new VisorSnapshotCheckTaskArg(snpName, snpPath); + } + /** {@inheritDoc} */ @Override public void printUsage(Logger log) { - usage(log, "Check snapshot:", SNAPSHOT, generalUsageOptions(), name(), SNAPSHOT_NAME_ARG); + Map params = new LinkedHashMap<>(generalUsageOptions()); + + params.put(SOURCE.optionName(), SOURCE.description()); + + usage(log, "Check snapshot:", SNAPSHOT, params, name(), SNAPSHOT_NAME_ARG, + optional(SOURCE.argName(), SOURCE.optionName())); } /** {@inheritDoc} */ diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java new file mode 100644 index 00000000000000..73d2b8616c85de --- /dev/null +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.commandline.snapshot; + +import org.apache.ignite.internal.commandline.argument.CommandArg; + +/** + * Snapshot check command options. + */ +public enum SnapshotCheckCommandOption implements CommandArg { + /** Snapshot directory location. */ + SOURCE("--src", "path", "Path to the directory where the snapshot files are located. If not specified, " + + "the default snapshot directory will be used."); + + /** Option name. */ + private final String argName; + + /** Option name. */ + private final String optionName; + + /** Option description. */ + private final String desc; + + /** + * @param argName Option name. + * @param desc Option description. + */ + SnapshotCheckCommandOption(String argName, String optionName, String desc) { + this.argName = argName; + this.optionName = optionName; + this.desc = desc; + } + + /** {@inheritDoc} */ + @Override public String argName() { + return argName; + } + + /** @return Option name. */ + public String optionName() { + return optionName; + } + + /** @return Option description. */ + public String description() { + return desc; + } + + /** @return Argument name. */ + @Override public String toString() { + return argName(); + } +} diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java index 97e4d22c83294d..0cd6eb16c17294 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java @@ -56,7 +56,7 @@ protected SnapshotCreateCommand() { "Possible options: " + F.concat(F.asList(SnapshotCreateCommandOption.values()), ", ") + '.'); } else if (option == DESTINATION) { - String errMsg = "Expected path to snapshot directory."; + String errMsg = "Expected path to the snapshot directory."; if (CommandArgIterator.isCommandOrOption(argIter.peekNextArg())) throw new IllegalArgumentException(errMsg); diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java index 9bb173d310d0c5..5154d8a1dcb7d1 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java @@ -28,7 +28,7 @@ public enum SnapshotCreateCommandOption implements CommandArg { "Otherwise, it will be performed in the background, and the command will immediately return control."), /** Snapshot directory path. */ - DESTINATION("--dest", "path", "The path to directory where the snapshot will be saved. If not specified, " + + DESTINATION("--dest", "path", "Path to the directory where the snapshot will be saved. If not specified, " + "the default snapshot directory will be used."); /** Option name. */ diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java index eff09eb52ae142..4f946523ba76e1 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java @@ -87,7 +87,7 @@ else if (option == GROUPS) { throw new IllegalArgumentException("Expected " + argDesc); } else if (option == SOURCE) { - String errMsg = "Expected path to snapshot directory."; + String errMsg = "Expected path to the snapshot directory."; if (CommandArgIterator.isCommandOrOption(argIter.peekNextArg())) throw new IllegalArgumentException(errMsg); diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java index 3520de7305649c..ed9146c3f24058 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java @@ -27,7 +27,7 @@ public enum SnapshotRestoreCommandOption implements CommandArg { GROUPS("--groups", "group1,...groupN", "Cache group names."), /** Snapshot directory location. */ - SOURCE("--src", "path", "The path to the directory where the snapshot files are located. If not specified, " + + SOURCE("--src", "path", "Path to the directory where the snapshot files are located. If not specified, " + "the default snapshot directory will be used."), /** Synchronous execution flag. */ diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 9e33cda79f5a89..d1f7e3985e5801 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3350,7 +3350,7 @@ public void testSnapshotRestore() throws Exception { /** @throws Exception If fails. */ @Test - public void testSnapshotCreateRestoreCustomDir() throws Exception { + public void testSnapshotCreateCheckAndRestoreCustomDir() throws Exception { int keysCnt = 100; File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); @@ -3377,6 +3377,13 @@ public void testSnapshotCreateRestoreCustomDir() throws Exception { assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "restore", snpName, "--start", "--sync")); assertContains(log, testOut.toString(), "Snapshot does not exists [snapshot=" + snpName); + // The check command simply prints the results of the check, it always ends with a zero exit code. + assertEquals(EXIT_CODE_OK, execute(h, "--snapshot", "check", snpName)); + assertContains(log, testOut.toString(), "Snapshot does not exists [snapshot=" + snpName); + + assertEquals(EXIT_CODE_OK, execute(h, "--snapshot", "check", snpName, "--src", snpDir.getAbsolutePath())); + assertContains(log, testOut.toString(), "The check procedure has finished, no conflicts have been found."); + assertEquals(EXIT_CODE_OK, execute(h, "--snapshot", "restore", snpName, "--start", "--sync", "--src", snpDir.getAbsolutePath())); @@ -3386,7 +3393,8 @@ public void testSnapshotCreateRestoreCustomDir() throws Exception { for (int i = 0; i < keysCnt; i++) assertEquals(Integer.valueOf(i), cache.get(i)); - } finally { + } + finally { U.delete(snpDir); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java index 62acaf0a7f4d16..339bf303f54ad3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; -import java.io.File; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 181d4a3e7f5bfc..68f3101d194597 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -67,7 +67,6 @@ import java.util.function.Consumer; import java.util.function.Predicate; import java.util.stream.Collectors; -import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteInterruptedException; @@ -1181,16 +1180,17 @@ public IgniteFuture cancelLocalRestoreTask(String name) { /** * @param name Snapshot name. + * @param snpPath Snapshot directory path. * @return Future with the result of execution snapshot partitions verify task, which besides calculating partition * hashes of {@link IdleVerifyResultV2} also contains the snapshot metadata distribution across the cluster. */ - public IgniteInternalFuture checkSnapshot(String name) { + public IgniteInternalFuture checkSnapshot(String name, @Nullable String snpPath) { A.notNullOrEmpty(name, "Snapshot name cannot be null or empty."); A.ensure(U.alphanumericUnderscore(name), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_"); cctx.kernalContext().security().authorize(ADMIN_SNAPSHOT); - return checkSnapshot(name, null, null, false).chain(f -> { + return checkSnapshot(name, snpPath, null, false).chain(f -> { try { return f.get().idleVerifyResult(); } @@ -1570,6 +1570,8 @@ public IgniteFuture restoreSnapshot(String name, @Nullable String snpPath, A.ensure(U.alphanumericUnderscore(name), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_"); A.ensure(grpNames == null || !grpNames.isEmpty(), "List of cache group names cannot be empty."); + cctx.kernalContext().security().authorize(ADMIN_SNAPSHOT); + return restoreCacheGrpProc.start(name, snpPath, grpNames); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java index 94e9bf536d695c..e1cb2570380987 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java @@ -118,11 +118,12 @@ private static class SnapshotHandlerRestoreJob extends ComputeJobAdapter { /** Cache group names. */ private final Collection grps; + /** Snapshot directory path. */ private final String snpPath; /** * @param snpName Snapshot name. - * @param snpPath todo + * @param snpPath Snapshot directory path. * @param consistentId String representation of the consistent node ID. * @param grps Cache group names. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java index 08bf1c838e00b9..6c20c6d2910d05 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; -import java.io.File; import java.io.Serializable; import java.util.Collection; import java.util.Set; @@ -41,7 +40,7 @@ public class SnapshotOperationRequest implements Serializable { /** Snapshot name. */ private final String snpName; - /** Snapshot path. */ + /** Snapshot directory path. */ private final String snpPath; /** Baseline node IDs that must be alive to complete the operation. */ @@ -100,7 +99,7 @@ public String snapshotName() { } /** - * @return Snapshot path. + * @return Snapshot directory path. */ public String snapshotPath() { return snpPath; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTaskArg.java index a26c8d0cd1a8d7..bcf6343a2a584b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTaskArg.java @@ -16,7 +16,6 @@ */ package org.apache.ignite.internal.processors.cache.persistence.snapshot; -import java.io.File; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; @@ -42,6 +41,7 @@ public class SnapshotPartitionsVerifyTaskArg extends VisorDataTransferObject { /** The map of distribution of snapshot metadata pieces across the cluster. */ private Map> clusterMetas; + /** Snapshot directory path. */ private String snpPath; /** Default constructor. */ @@ -52,7 +52,7 @@ public SnapshotPartitionsVerifyTaskArg() { /** * @param grpNames Cache group names to be verified. * @param clusterMetas The map of distribution of snapshot metadata pieces across the cluster. - * @param snpPath todo + * @param snpPath Snapshot directory path. */ public SnapshotPartitionsVerifyTaskArg( Collection grpNames, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 5f2019da643ed4..006b13deb5355d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -304,10 +304,6 @@ public IgniteFuture start(String snpName, @Nullable String snpPath, @Nulla cacheGrpNames.stream().collect(Collectors.toMap(CU::cacheId, v -> v)); for (Map.Entry> entry : metas.entrySet()) { - // todo remove - if (entry.getKey().id().toString().endsWith("2")) - continue; - dataNodes.add(entry.getKey().id()); for (SnapshotMetadata meta : entry.getValue()) { @@ -587,9 +583,7 @@ private IgniteInternalFuture prepare(SnapshotO ", caches=" + req.groups() + ']'); } - List locMetas = snpMgr.readSnapshotMetadatas(req.snapshotName(), !ctx.localNodeId().toString().endsWith("2") ? req.snapshotPath() : null); - -// !ctx.localNodeId().toString().endsWith("2") ? req.snapshotPath() : null); + List locMetas = snpMgr.readSnapshotMetadatas(req.snapshotName(), req.snapshotPath()); SnapshotRestoreContext opCtx0 = prepareContext(req, locMetas); @@ -881,7 +875,6 @@ private IgniteInternalFuture preload(UUID reqId) { Map> allParts = new HashMap<>(); Map> rmtLoadParts = new HashMap<>(); ClusterNode locNode = ctx.cache().context().localNode(); - //ctx.localNodeId().toString().endsWith("2") ? Collections.emptyList() : List locMetas = opCtx0.metasPerNode.get(locNode.id()); // First preload everything from the local node. @@ -904,14 +897,8 @@ private IgniteInternalFuture preload(UUID reqId) { availParts.addAll(parts); } - if (ctx.localNodeId().toString().endsWith("2")) - System.out.println(">xxx> Cache = " + cacheOrGrpName + ", availParts =" + availParts); - List> assignment = affCache.get(cacheOrGrpName).idealAssignment().assignment(); -// if (ctx.localNodeId().toString().endsWith("2")) -// System.out.println(">xxx> assignment " + assignment); - Set partFuts = availParts .stream() .filter(p -> p != INDEX_PARTITION && assignment.get(p).contains(locNode)) @@ -919,10 +906,6 @@ private IgniteInternalFuture preload(UUID reqId) { .collect(Collectors.toSet()); allParts.put(grpId, partFuts); - - if (ctx.localNodeId().toString().endsWith("2")) - System.out.println(">xxx> partFuts =" + partFuts); - rmtLoadParts.put(grpId, leftParts = new HashSet<>(partFuts)); if (leftParts.isEmpty()) @@ -982,8 +965,6 @@ private IgniteInternalFuture preload(UUID reqId) { } } - System.out.println(">xxx> rmtLoadParts " + rmtLoadParts); - // Load other partitions from remote nodes. List rmtAwaitParts = rmtLoadParts.values().stream() .flatMap(Collection::stream) @@ -1406,6 +1387,7 @@ private static class SnapshotRestoreContext { /** Snapshot name. */ private final String snpName; + /** Snapshot directory path. */ private final String snpPath; /** Baseline discovery cache for node IDs that must be alive to complete the operation.*/ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCheckTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCheckTask.java index 5bd01d0d4fadb2..202fea06de8c5b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCheckTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCheckTask.java @@ -25,34 +25,36 @@ import org.apache.ignite.internal.visor.VisorJob; /** - * @see IgniteSnapshotManager#checkSnapshot(String) + * @see IgniteSnapshotManager#checkSnapshot(String, String) */ @GridInternal -public class VisorSnapshotCheckTask extends VisorSnapshotOneNodeTask { +public class VisorSnapshotCheckTask extends VisorSnapshotOneNodeTask { /** Serial version uid. */ private static final long serialVersionUID = 0L; /** {@inheritDoc} */ - @Override protected VisorJob job(String arg) { + @Override protected VisorJob job(VisorSnapshotCheckTaskArg arg) { return new VisorSnapshotCheckJob(arg, debug); } /** */ - private static class VisorSnapshotCheckJob extends VisorJob { + private static class VisorSnapshotCheckJob extends VisorJob { /** Serial version uid. */ private static final long serialVersionUID = 0L; /** - * @param name Snapshot name. + * @param arg Snapshot check task argument. * @param debug Flag indicating whether debug information should be printed into node log. */ - protected VisorSnapshotCheckJob(String name, boolean debug) { - super(name, debug); + protected VisorSnapshotCheckJob(VisorSnapshotCheckTaskArg arg, boolean debug) { + super(arg, debug); } /** {@inheritDoc} */ - @Override protected IdleVerifyResultV2 run(String name) throws IgniteException { - return new IgniteFutureImpl<>(ignite.context().cache().context().snapshotMgr().checkSnapshot(name)) + @Override protected IdleVerifyResultV2 run(VisorSnapshotCheckTaskArg arg) throws IgniteException { + IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); + + return new IgniteFutureImpl<>(snpMgr.checkSnapshot(arg.snapshotName(), arg.snapshotPath())) .get(); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCheckTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCheckTaskArg.java new file mode 100644 index 00000000000000..c3e708ce1165b4 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCheckTaskArg.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.snapshot; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; + +/** + * Argument for the task to check snapshot. + */ +public class VisorSnapshotCheckTaskArg extends IgniteDataTransferObject { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Snapshot name. */ + private String snpName; + + /** Snapshot directory path. */ + private String snpPath; + + /** Default constructor. */ + public VisorSnapshotCheckTaskArg() { + // No-op. + } + + /** + * @param snpName Snapshot name. + * @param snpPath Snapshot directory path. + */ + public VisorSnapshotCheckTaskArg(String snpName, String snpPath) { + this.snpName = snpName; + this.snpPath = snpPath; + } + + /** @return Snapshot name. */ + public String snapshotName() { + return snpName; + } + + /** @return Snapshot directory path. */ + public String snapshotPath() { + return snpPath; + } + + /** {@inheritDoc} */ + @Override protected void writeExternalData(ObjectOutput out) throws IOException { + U.writeString(out, snpName); + U.writeString(out, snpPath); + } + + /** {@inheritDoc} */ + @Override protected void readExternalData(byte ver, ObjectInput in) throws IOException, ClassNotFoundException { + snpName = U.readString(in); + snpPath = U.readString(in); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(VisorSnapshotCheckTaskArg.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTaskArg.java index e1a9ba00687f1f..a828cf9ec9d9e8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotCreateTaskArg.java @@ -20,23 +20,15 @@ import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; -import org.apache.ignite.internal.dto.IgniteDataTransferObject; import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.internal.util.typedef.internal.U; /** * Argument for the task to create snapshot. */ -public class VisorSnapshotCreateTaskArg extends IgniteDataTransferObject { +public class VisorSnapshotCreateTaskArg extends VisorSnapshotCheckTaskArg { /** Serial version uid. */ private static final long serialVersionUID = 0L; - /** Snapshot name. */ - private String snpName; - - /** Snapshot directory path. */ - private String snpPath; - /** Synchronous execution flag. */ private boolean sync; @@ -51,19 +43,9 @@ public VisorSnapshotCreateTaskArg() { * @param sync Synchronous execution flag. */ public VisorSnapshotCreateTaskArg(String snpName, String snpPath, boolean sync) { - this.snpName = snpName; - this.snpPath = snpPath; - this.sync = sync; - } + super(snpName, snpPath); - /** @return Snapshot name. */ - public String snapshotName() { - return snpName; - } - - /** @return Snapshot directory path. */ - public String snapshotPath() { - return snpPath; + this.sync = sync; } /** @return Synchronous execution flag. */ @@ -73,15 +55,15 @@ public boolean sync() { /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { - U.writeString(out, snpName); - U.writeString(out, snpPath); + super.writeExternalData(out); + out.writeBoolean(sync); } /** {@inheritDoc} */ @Override protected void readExternalData(byte ver, ObjectInput in) throws IOException, ClassNotFoundException { - snpName = U.readString(in); - snpPath = U.readString(in); + super.readExternalData(ver, in); + sync = in.readBoolean(); } diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index 5aeb89669a7c3e..ebeac9d379122e 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -2285,6 +2285,7 @@ org.apache.ignite.internal.visor.snapshot.VisorSnapshotRestoreTask org.apache.ignite.internal.visor.snapshot.VisorSnapshotRestoreTask$VisorSnapshotStartRestoreJob org.apache.ignite.internal.visor.snapshot.VisorSnapshotRestoreTask$VisorSnapshotRestoreCancelJob org.apache.ignite.internal.visor.snapshot.VisorSnapshotRestoreTask$VisorSnapshotRestoreStatusJob +org.apache.ignite.internal.visor.snapshot.VisorSnapshotCheckTaskArg org.apache.ignite.internal.visor.snapshot.VisorSnapshotCreateTaskArg org.apache.ignite.internal.visor.snapshot.VisorSnapshotRestoreTaskArg org.apache.ignite.internal.visor.snapshot.VisorSnapshotRestoreTaskAction diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java index e4ebdeca113d05..8912e8e565e71a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java @@ -194,7 +194,7 @@ public void testValidatingSnapshotFailsWithOtherMasterKey() throws Exception { ig = startGrids(2); - IdleVerifyResultV2 snpCheckRes = snp(ig).checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 snpCheckRes = snp(ig).checkSnapshot(SNAPSHOT_NAME, null).get(); for (Exception e : snpCheckRes.exceptions().values()) { if (e.getMessage().contains("different master key digest")) @@ -236,7 +236,7 @@ public void testValidatingSnapshotFailsWithNoEncryption() throws Exception { ig.cluster().state(ACTIVE); - IdleVerifyResultV2 snpCheckRes = snp(ig).checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 snpCheckRes = snp(ig).checkSnapshot(SNAPSHOT_NAME, null).get(); for (Exception e : snpCheckRes.exceptions().values()) { if (e.getMessage().contains("has encrypted caches while encryption is disabled")) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java index bce14dac639b57..d477e2533c6de0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java @@ -122,7 +122,7 @@ public void testClusterSnapshotCheck() throws Exception { ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(); - IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, null).get(); StringBuilder b = new StringBuilder(); res.print(b::append, true); @@ -147,7 +147,7 @@ public void testClusterSnapshotCheckMissedPart() throws Exception { assertTrue(part0.toString(), part0.toFile().exists()); assertTrue(part0.toFile().delete()); - IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, null).get(); StringBuilder b = new StringBuilder(); res.print(b::append, true); @@ -172,7 +172,7 @@ public void testClusterSnapshotCheckMissedGroup() throws Exception { assertTrue(dir.toString(), dir.toFile().exists()); assertTrue(U.delete(dir)); - IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, null).get(); StringBuilder b = new StringBuilder(); res.print(b::append, true); @@ -196,7 +196,7 @@ public void testClusterSnapshotCheckMissedMeta() throws Exception { assertTrue(smfs[0].toString(), smfs[0].exists()); assertTrue(U.delete(smfs[0])); - IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, null).get(); StringBuilder b = new StringBuilder(); res.print(b::append, true); @@ -217,7 +217,7 @@ public void testClusterSnapshotCheckWithNodeFilter() throws Exception { ig0.snapshot().createSnapshot(SNAPSHOT_NAME).get(); - IdleVerifyResultV2 res = snp(ig0).checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 res = snp(ig0).checkSnapshot(SNAPSHOT_NAME, null).get(); StringBuilder b = new StringBuilder(); res.print(b::append, true); @@ -270,7 +270,7 @@ public void testClusterSnapshotCheckPartitionCounters() throws Exception { pageStore.finishRecover(); } - IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, null).get(); StringBuilder b = new StringBuilder(); res.print(b::append, true); @@ -318,7 +318,7 @@ public void testClusterSnapshotCheckOtherCluster() throws Exception { ignite.cluster().baselineAutoAdjustEnabled(false); ignite.cluster().state(ACTIVE); - IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, null).get(); StringBuilder b = new StringBuilder(); res.print(b::append, true); @@ -342,7 +342,7 @@ public void testClusterSnapshotCheckCRCFail() throws Exception { corruptPartitionFile(ignite, SNAPSHOT_NAME, dfltCacheCfg, PART_ID); - IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, null).get(); StringBuilder b = new StringBuilder(); res.print(b::append, true); @@ -448,7 +448,7 @@ public void testClusterSnapshotCheckFailsOnPartitionDataDiffers() throws Excepti assertNotNull(part0); assertTrue(part0.toString(), part0.toFile().exists()); - IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, null).get(); StringBuilder b = new StringBuilder(); res.print(b::append, true); @@ -548,12 +548,12 @@ public void testClusterSnapshotCheckMultipleTimes() throws Exception { // Warmup. for (int i = 0; i < iterations; i++) - snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(); + snp(ignite).checkSnapshot(SNAPSHOT_NAME, null).get(); int activeThreadsCntBefore = Thread.activeCount(); for (int i = 0; i < iterations; i++) - snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(); + snp(ignite).checkSnapshot(SNAPSHOT_NAME, null).get(); int createdThreads = Thread.activeCount() - activeThreadsCntBefore; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index a4aaec360e4caa..85b56029415381 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -120,65 +120,39 @@ public void testRestoreWithEmptyPartitions() throws Exception { assertCacheKeys(ignite.cache(DEFAULT_CACHE_NAME), keysCnt); } + /** + * @throws Exception If failed. + */ @Test - public void testClusterSnapshotRestoreWithExplicitPath() throws Exception { + public void testClusterSnapshotRestoreFromCustomDir() throws Exception { File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); assert snpDir.list().length == 0 : "Target directory is not empty: " + Arrays.asList(snpDir.list()); try { - IgniteEx ignite = null; - -// dfltCacheCfg.setBackups(0); - dfltCacheCfg.setCacheMode(CacheMode.REPLICATED); - dfltCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 8)); - - for (int i = 0; i < 2; i++) { -// IgniteConfiguration cfg = optimize(getConfiguration(getTestIgniteInstanceName(i))); - -// cfg.setCacheConfiguration((CacheConfiguration[])null); -// ignite = startGrid(i); - -// if (cfgPath) -// cfg.setSnapshotPath(snpDir.getAbsolutePath()); - - ignite = startGrid(i); - } - -// ignite.cluster().baselineAutoAdjustEnabled(false); + IgniteEx ignite = startGrids(2); ignite.cluster().state(ACTIVE); for (int i = 0; i < CACHE_KEYS_RANGE; i++) ignite.cache(DEFAULT_CACHE_NAME).put(i, i); -// IgniteSnapshotManager snpMgr = ; - ignite.context().cache().context().snapshotMgr().createSnapshot(SNAPSHOT_NAME, snpDir.toString()).get(TIMEOUT); -// IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, snpDir, null, false).get(TIMEOUT); -// -// StringBuilder sb = new StringBuilder(); -// res.print(sb::append, true); -// -// assertTrue(F.isEmpty(res.exceptions())); -// assertPartitionsSame(res); -// assertContains(log, sb.toString(), "The check procedure has finished, no conflicts have been found"); - - ignite = startGrid(2); -// ignite.cluster().state(ACTIVE); - resetBaselineTopology(); + // Check snapshot. + IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, snpDir.getAbsolutePath()).get(TIMEOUT); - awaitPartitionMapExchange(); + StringBuilder sb = new StringBuilder(); + res.print(sb::append, true); + assertTrue(F.isEmpty(res.exceptions())); + assertPartitionsSame(res); + assertContains(log, sb.toString(), "The check procedure has finished, no conflicts have been found"); ignite.destroyCache(DEFAULT_CACHE_NAME); awaitPartitionMapExchange(); - ignite.context().cache().context().snapshotMgr().restoreSnapshot(SNAPSHOT_NAME, snpDir.toString(), null).get(); - -// stopAllGrids(); - -// IgniteEx snp = startGridsFromSnapshot(2, cfg -> snpDir.getAbsolutePath(), SNAPSHOT_NAME, true); + ignite.context().cache().context().snapshotMgr().restoreSnapshot(SNAPSHOT_NAME, snpDir.getAbsolutePath(), null) + .get(); IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); @@ -191,7 +165,6 @@ public void testClusterSnapshotRestoreWithExplicitPath() throws Exception { U.delete(snpDir); } - } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java index 30a87d19ee8f20..98ffb8be1e062d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java @@ -36,7 +36,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; -import java.util.function.Function; import java.util.function.Predicate; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRestoreFromRemoteTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRestoreFromRemoteTest.java index fa75b5628f8718..3c4d3ae3221db9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRestoreFromRemoteTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRestoreFromRemoteTest.java @@ -196,7 +196,7 @@ public void testRestoreFromAnEmptyNode() throws Exception { // Ensure that the snapshot check command succeeds. IdleVerifyResultV2 res = - emptyNode.context().cache().context().snapshotMgr().checkSnapshot(SNAPSHOT_NAME).get(TIMEOUT); + emptyNode.context().cache().context().snapshotMgr().checkSnapshot(SNAPSHOT_NAME, null).get(TIMEOUT); StringBuilder buf = new StringBuilder(); res.print(buf::append, true); diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output index b18f60f0f3d047..07271c9fd36724 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output @@ -144,7 +144,7 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. - path - The path to directory where the snapshot will be saved. If not specified, the default snapshot directory will be used. + path - Path to the directory where the snapshot will be saved. If not specified, the default snapshot directory will be used. sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Cancel running snapshot: @@ -154,10 +154,11 @@ This utility can do the following commands: snapshot_name - Snapshot name. Check snapshot: - control.(sh|bat) --snapshot check snapshot_name + control.(sh|bat) --snapshot check snapshot_name [--src path] Parameters: snapshot_name - Snapshot name. + path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. Restore snapshot: control.(sh|bat) --snapshot restore snapshot_name --start [--groups group1,...groupN] [--src path] [--sync] @@ -165,7 +166,7 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. group1,...groupN - Cache group names. - path - The path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. + path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Snapshot restore operation status: diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output index b18f60f0f3d047..07271c9fd36724 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output @@ -144,7 +144,7 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. - path - The path to directory where the snapshot will be saved. If not specified, the default snapshot directory will be used. + path - Path to the directory where the snapshot will be saved. If not specified, the default snapshot directory will be used. sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Cancel running snapshot: @@ -154,10 +154,11 @@ This utility can do the following commands: snapshot_name - Snapshot name. Check snapshot: - control.(sh|bat) --snapshot check snapshot_name + control.(sh|bat) --snapshot check snapshot_name [--src path] Parameters: snapshot_name - Snapshot name. + path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. Restore snapshot: control.(sh|bat) --snapshot restore snapshot_name --start [--groups group1,...groupN] [--src path] [--sync] @@ -165,7 +166,7 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. group1,...groupN - Cache group names. - path - The path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. + path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Snapshot restore operation status: diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckWithIndexesTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckWithIndexesTest.java index cd2b0caa5f40b4..54819c6bce5128 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckWithIndexesTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckWithIndexesTest.java @@ -44,7 +44,7 @@ public void testClusterSnapshotCheckEmptyCache() throws Exception { ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT); - IdleVerifyResultV2 res = ignite.context().cache().context().snapshotMgr().checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 res = ignite.context().cache().context().snapshotMgr().checkSnapshot(SNAPSHOT_NAME, null).get(); StringBuilder b = new StringBuilder(); res.print(b::append, true); @@ -61,7 +61,7 @@ public void testClusterSnapshotCheckWithIndexes() throws Exception { ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT); - IdleVerifyResultV2 res = ignite.context().cache().context().snapshotMgr().checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 res = ignite.context().cache().context().snapshotMgr().checkSnapshot(SNAPSHOT_NAME, null).get(); StringBuilder b = new StringBuilder(); res.print(b::append, true); @@ -87,7 +87,7 @@ public void testClusterSnapshotCheckWithNodeFilter() throws Exception { grid(0).snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT); - IdleVerifyResultV2 res = grid(0).context().cache().context().snapshotMgr().checkSnapshot(SNAPSHOT_NAME).get(); + IdleVerifyResultV2 res = grid(0).context().cache().context().snapshotMgr().checkSnapshot(SNAPSHOT_NAME, null).get(); StringBuilder b = new StringBuilder(); res.print(b::append, true); From f9338b05a0a1bff7c3d8fdf584940284fede69dd Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Mon, 30 May 2022 19:57:30 +0300 Subject: [PATCH 08/19] IGNITE-15067 (minor) Test fixes + code cleanup. --- .../snapshot/SnapshotRestoreCommandOption.java | 4 ++-- .../apache/ignite/util/GridCommandHandlerTest.java | 12 +++++------- .../persistence/snapshot/IgniteSnapshotManager.java | 12 ++++++------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java index ed9146c3f24058..a0532ac2519aff 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java @@ -27,8 +27,8 @@ public enum SnapshotRestoreCommandOption implements CommandArg { GROUPS("--groups", "group1,...groupN", "Cache group names."), /** Snapshot directory location. */ - SOURCE("--src", "path", "Path to the directory where the snapshot files are located. If not specified, " + - "the default snapshot directory will be used."), + SOURCE(SnapshotCheckCommandOption.SOURCE.argName(), SnapshotCheckCommandOption.SOURCE.optionName(), + SnapshotCheckCommandOption.SOURCE.description()), /** Synchronous execution flag. */ SYNC(SnapshotCreateCommandOption.SYNC.argName(), SnapshotCreateCommandOption.SYNC.optionName(), diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index d1f7e3985e5801..3cf17258ab2848 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3075,7 +3075,7 @@ private void doClusterSnapshotCreate(boolean syncMode) throws Exception { // Invalid command syntax check. assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "create", snpName, "blah")); - assertContains(log, testOut.toString(), "Invalid argument: blah. Possible options: --sync."); + assertContains(log, testOut.toString(), "Invalid argument: blah. Possible options: --sync, --dest."); assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "create", snpName, "--sync", "blah")); assertContains(log, testOut.toString(), "Invalid argument: blah."); @@ -3205,7 +3205,7 @@ public void testSnapshotRestoreSynchronously() throws Exception { assertContains(log, testOut.toString(), "Invalid argument: --sync."); assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "restore", snpName, "--start", "blah")); - assertContains(log, testOut.toString(), "Invalid argument: blah. Possible options: --groups, --sync."); + assertContains(log, testOut.toString(), "Invalid argument: blah. Possible options: --groups, --src, --sync."); autoConfirmation = true; @@ -3269,7 +3269,8 @@ public void testSnapshotRestore() throws Exception { CommandHandler h = new CommandHandler(); assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "restore", snpName, "--start", cacheName1)); - assertContains(log, testOut.toString(), "Invalid argument: " + cacheName1 + ". Possible options: --groups, --sync."); + assertContains(log, testOut.toString(), + "Invalid argument: " + cacheName1 + ". Possible options: --groups, --src, --sync."); // Restore single cache group. assertEquals(EXIT_CODE_OK, execute(h, "--snapshot", "restore", snpName, "--start", "--groups", cacheName1)); @@ -3352,13 +3353,11 @@ public void testSnapshotRestore() throws Exception { @Test public void testSnapshotCreateCheckAndRestoreCustomDir() throws Exception { int keysCnt = 100; - + String snpName = "snapshot_30052022"; File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); assert snpDir.list().length == 0 : "Target directory is not empty: " + Arrays.asList(snpDir.list()); - String snpName = "snapshot_30052022"; - try { Ignite ignite = startGrids(2); ignite.cluster().state(ACTIVE); @@ -3366,7 +3365,6 @@ public void testSnapshotCreateCheckAndRestoreCustomDir() throws Exception { createCacheAndPreload(ignite, keysCnt); injectTestSystemOut(); - CommandHandler h = new CommandHandler(); assertEquals(EXIT_CODE_OK, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 68f3101d194597..3701853c40a7a8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -698,14 +698,14 @@ public File snapshotLocalDir(String snpName) { /** * @param snpName Snapshot name. - * @param snpLocation Snapshot directory path. + * @param snpPath Snapshot directory path. * @return Local snapshot directory for snapshot with given name. */ - public File snapshotLocalDir(String snpName, @Nullable String snpLocation) { + public File snapshotLocalDir(String snpName, @Nullable String snpPath) { assert locSnpDir != null; assert U.alphanumericUnderscore(snpName) : snpName; - return snpLocation == null ? new File(locSnpDir, snpName) : new File(snpLocation, snpName); + return snpPath == null ? new File(locSnpDir, snpName) : new File(snpPath, snpName); } /** @@ -1372,16 +1372,16 @@ private SnapshotMetadata readSnapshotMetadata(File smf) { /** * @param snpName Snapshot name. - * @param snpLocation Snapshot directory path. + * @param snpPath Snapshot directory path. * @return List of snapshot metadata for the given snapshot name on local node. * If snapshot has been taken from local node the snapshot metadata for given * local node will be placed on the first place. */ - public List readSnapshotMetadatas(String snpName, @Nullable String snpLocation) { + public List readSnapshotMetadatas(String snpName, @Nullable String snpPath) { A.notNullOrEmpty(snpName, "Snapshot name cannot be null or empty."); A.ensure(U.alphanumericUnderscore(snpName), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_"); - File snpDir = snapshotLocalDir(snpName, snpLocation); + File snpDir = snapshotLocalDir(snpName, snpPath); if (!(snpDir.exists() && snpDir.isDirectory())) return Collections.emptyList(); From 0eef7602f11f48743a9a1cd300478d1d4f68dde1 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Mon, 30 May 2022 20:24:13 +0300 Subject: [PATCH 09/19] IGNITE-15067 (minor) Handler context test. --- .../IgniteClusterSnapshotHandlerTest.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java index 872b6b97b1c804..99bb718433c366 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java @@ -349,4 +349,60 @@ public void testCrdChangeDuringHandlerCompleteOnSnapshotCreate() throws Exceptio startGrid(0); grid(0).snapshot().createSnapshot(SNAPSHOT_NAME); } + + /** + * Test ensures that the location of the snapshot is valid. + * + * @throws Exception If failed. + */ + @Test + public void testHandlerSnapshotLocation() throws Exception { + String snpName = "snapshot_30052022"; + File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); + String expFullPath = new File(snpDir, snpName).getAbsolutePath(); + + SnapshotHandler createHnd = new SnapshotHandler() { + @Override public SnapshotHandlerType type() { + return SnapshotHandlerType.CREATE; + } + + @Override public Void invoke(SnapshotHandlerContext ctx) { + if (!expFullPath.equals(ctx.snapshotDirectory().getAbsolutePath())) + throw new IllegalStateException("Expected " + expFullPath + ", actual " + ctx.snapshotDirectory()); + + return null; + } + }; + + SnapshotHandler restoreHnd = new SnapshotHandler() { + @Override public SnapshotHandlerType type() { + return SnapshotHandlerType.RESTORE; + } + + @Override public Void invoke(SnapshotHandlerContext ctx) throws Exception { + createHnd.invoke(ctx); + + return null; + } + }; + + handlers.add(createHnd); + handlers.add(restoreHnd); + + try { + IgniteEx ignite = startGridsWithCache(1, CACHE_KEYS_RANGE, valueBuilder(), dfltCacheCfg); + + IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); + + snpMgr.createSnapshot(snpName, snpDir.getAbsolutePath()).get(TIMEOUT); + + ignite.destroyCache(DEFAULT_CACHE_NAME); + awaitPartitionMapExchange(); + + snpMgr.restoreSnapshot(snpName, snpDir.getAbsolutePath(), null).get(TIMEOUT); + } + finally { + U.delete(snpDir); + } + } } From c19e89661470eb35cb650649e2323d6eea49c550 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Tue, 31 May 2022 10:45:45 +0300 Subject: [PATCH 10/19] IGNITE-15067 Added MXBean param. --- .../persistence/snapshot/SnapshotMXBeanImpl.java | 8 ++++---- .../org/apache/ignite/mxbean/SnapshotMXBean.java | 11 ++++++++++- .../snapshot/IgniteSnapshotMXBeanTest.java | 12 ++---------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java index 60abf07606ae66..33389881a26f7f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java @@ -40,8 +40,8 @@ public SnapshotMXBeanImpl(GridKernalContext ctx) { } /** {@inheritDoc} */ - @Override public void createSnapshot(String snpName) { - IgniteFuture fut = mgr.createSnapshot(snpName); + @Override public void createSnapshot(String snpName, String snpPath) { + IgniteFuture fut = mgr.createSnapshot(snpName, F.isEmpty(snpPath) ? null : snpPath); if (fut.isDone()) fut.get(); @@ -53,11 +53,11 @@ public SnapshotMXBeanImpl(GridKernalContext ctx) { } /** {@inheritDoc} */ - @Override public void restoreSnapshot(String name, String grpNames) { + @Override public void restoreSnapshot(String name, String path, String grpNames) { Set grpNamesSet = F.isEmpty(grpNames) ? null : Arrays.stream(grpNames.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toSet()); - IgniteFuture fut = mgr.restoreSnapshot(name, grpNamesSet); + IgniteFuture fut = mgr.restoreSnapshot(name, F.isEmpty(path) ? null : path, grpNamesSet); if (fut.isDone()) fut.get(); diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java index fcb7a399c98783..b18ac51dc1b043 100644 --- a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java +++ b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java @@ -29,10 +29,16 @@ public interface SnapshotMXBean { * Create the cluster-wide snapshot with given name asynchronously. * * @param snpName Snapshot name to be created. + * @param snpPath Snapshot directory path. * @see IgniteSnapshot#createSnapshot(String) (String) */ @MXBeanDescription("Create cluster-wide snapshot.") - public void createSnapshot(@MXBeanParameter(name = "snpName", description = "Snapshot name.") String snpName); + public void createSnapshot( + @MXBeanParameter(name = "snpName", description = "Snapshot name.") + String snpName, + @MXBeanParameter(name = "snpPath", description = "Optional snapshot directory path.") + String snpPath + ); /** * Cancel previously started snapshot operation on the node initiator. @@ -46,6 +52,7 @@ public interface SnapshotMXBean { * Restore cluster-wide snapshot. * * @param name Snapshot name. + * @param path Snapshot directory path. * @param cacheGroupNames Optional comma-separated list of cache group names. * @see IgniteSnapshot#restoreSnapshot(String, Collection) */ @@ -53,6 +60,8 @@ public interface SnapshotMXBean { public void restoreSnapshot( @MXBeanParameter(name = "snpName", description = "Snapshot name.") String name, + @MXBeanParameter(name = "snpPath", description = "Optional snapshot directory path.") + String path, @MXBeanParameter(name = "cacheGroupNames", description = "Optional comma-separated list of cache group names.") String cacheGroupNames ); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java index 194008b568d73f..fd68a57b5f842d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java @@ -64,7 +64,7 @@ public void testCreateSnapshot() throws Exception { SnapshotMXBean mxBean = getMxBean(ignite.name(), METRIC_GROUP, SnapshotMXBeanImpl.class, SnapshotMXBean.class); - mxBean.createSnapshot(SNAPSHOT_NAME); + mxBean.createSnapshot(SNAPSHOT_NAME, ""); assertTrue("Waiting for snapshot operation failed.", GridTestUtils.waitForCondition(() -> (long)getMetric("LastSnapshotEndTime", snpMBean) > 0, TIMEOUT)); @@ -95,10 +95,6 @@ public void testCancelSnapshot() throws Exception { /** @throws Exception If fails. */ @Test public void testRestoreSnapshot() throws Exception { - // TODO IGNITE-14999 Support dynamic restoration of encrypted snapshots. - if (encryption) - return; - IgniteEx ignite = startGridsWithSnapshot(2, CACHE_KEYS_RANGE, false); DynamicMBean mReg0 = metricRegistry(grid(0).name(), null, SNAPSHOT_RESTORE_METRICS); @@ -108,7 +104,7 @@ public void testRestoreSnapshot() throws Exception { assertEquals(0, (long)getMetric("endTime", mReg1)); getMxBean(ignite.name(), METRIC_GROUP, SnapshotMXBeanImpl.class, SnapshotMXBean.class) - .restoreSnapshot(SNAPSHOT_NAME, null); + .restoreSnapshot(SNAPSHOT_NAME, "", ""); assertTrue(GridTestUtils.waitForCondition(() -> (long)getMetric("endTime", mReg0) > 0, TIMEOUT)); assertTrue(GridTestUtils.waitForCondition(() -> (long)getMetric("endTime", mReg1) > 0, TIMEOUT)); @@ -119,10 +115,6 @@ public void testRestoreSnapshot() throws Exception { /** @throws Exception If fails. */ @Test public void testCancelRestoreSnapshot() throws Exception { - // TODO IGNITE-14999 Support dynamic restoration of encrypted snapshots. - if (encryption) - return; - IgniteEx ignite = startGridsWithSnapshot(2, CACHE_KEYS_RANGE, false); SnapshotMXBean mxBean = getMxBean(ignite.name(), METRIC_GROUP, SnapshotMXBeanImpl.class, SnapshotMXBean.class); DynamicMBean mReg0 = metricRegistry(grid(0).name(), null, SNAPSHOT_RESTORE_METRICS); From f454b79acb662315c67288a87d7c3398c2be63e7 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Tue, 31 May 2022 12:54:33 +0300 Subject: [PATCH 11/19] IGNITE-15067 Fix snapshot view. --- .../cache/persistence/snapshot/IgniteSnapshotManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 3701853c40a7a8..3c0a207d79a060 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -2104,7 +2104,7 @@ private IgniteFuture executeRestoreManagementTask( * @return Snapshot view. */ private SnapshotView snapshotViewSupplier(SnapshotMetadata meta) { - List dirs = snapshotCacheDirectories(meta.snapshotName(), meta.folderName(), null, name -> true); + List dirs = snapshotCacheDirectories(meta.snapshotName(), null, meta.folderName(), name -> true); Collection cacheGrps = F.viewReadOnly(dirs, FilePageStoreManager::cacheGroupName); return new SnapshotView(meta.snapshotName(), meta.consistentId(), F.concat(meta.baselineNodes(), ","), F.concat(cacheGrps, ",")); From 806254c91fde12b2c399a4c9ef0c636cacdd7938 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Tue, 31 May 2022 13:18:08 +0300 Subject: [PATCH 12/19] IGNITE-15067 (minor) Code cleanup. --- .../snapshot/SnapshotCheckCommandOption.java | 3 ++- .../snapshot/SnapshotCreateCommandOption.java | 3 ++- .../snapshot/AbstractSnapshotVerificationTask.java | 2 +- .../persistence/snapshot/IgniteSnapshotManager.java | 10 +++------- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java index 73d2b8616c85de..52c463d66f0a72 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java @@ -37,7 +37,8 @@ public enum SnapshotCheckCommandOption implements CommandArg { private final String desc; /** - * @param argName Option name. + * @param argName Argument name. + * @param optionName Option name. * @param desc Option description. */ SnapshotCheckCommandOption(String argName, String optionName, String desc) { diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java index 5154d8a1dcb7d1..f5b6db20d44ab0 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java @@ -41,7 +41,8 @@ public enum SnapshotCreateCommandOption implements CommandArg { private final String desc; /** - * @param argName Option name. + * @param argName Argument name. + * @param optionName Option name. * @param desc Option description. */ SnapshotCreateCommandOption(String argName, String optionName, String desc) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java index 339bf303f54ad3..122775bc0a1b4a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java @@ -123,7 +123,7 @@ public static void checkMissedMetadata(Collection clusterMetas /** * @param name Snapshot name. - * @param snpPath todo + * @param snpPath Snapshot directory path. * @param constId Snapshot metadata file name. * @param groups Cache groups to be restored from the snapshot. May be empty if all cache groups are being restored. * @return Compute job. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 3c0a207d79a060..f98489e7d33162 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -276,14 +276,10 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter /** Maximum block size for limited snapshot transfer (64KB by default). */ public static final int SNAPSHOT_LIMITED_TRANSFER_BLOCK_SIZE_BYTES = 64 * 1024; - /** Metastorage key to save currently running snapshot directory. */ + /** Metastorage key to save currently running snapshot directory path. */ private static final String SNP_RUNNING_DIR_KEY = "snapshot-running-dir"; - /** - * Metastorage key to save currently running snapshot. - * - * @deprecated Use #SNP_RUNNING_DIR_KEY instead. - */ + /** @deprecated Use #SNP_RUNNING_DIR_KEY instead. */ @Deprecated private static final String SNP_RUNNING_KEY = "snapshot-running"; @@ -699,7 +695,7 @@ public File snapshotLocalDir(String snpName) { /** * @param snpName Snapshot name. * @param snpPath Snapshot directory path. - * @return Local snapshot directory for snapshot with given name. + * @return Local snapshot directory where snapshot files are located. */ public File snapshotLocalDir(String snpName, @Nullable String snpPath) { assert locSnpDir != null; From f18c8cb9fb365a677a414e57cacf090c0ea23f13 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Tue, 31 May 2022 14:23:23 +0300 Subject: [PATCH 13/19] IGNITE-15067 (minor) Code cleanup. --- .../ignite/util/GridCommandHandlerTest.java | 2 +- .../AbstractSnapshotVerificationTask.java | 4 +-- .../snapshot/IgniteSnapshotManager.java | 17 +++++------ .../snapshot/SnapshotFilesRequestMessage.java | 2 +- .../snapshot/SnapshotHandlerRestoreTask.java | 8 +++--- .../SnapshotPartitionsVerifyTask.java | 28 ++++++++++--------- .../SnapshotResponseRemoteFutureTask.java | 4 +-- .../snapshot/SnapshotRestoreProcess.java | 7 ++--- .../IgniteClusterSnapshotHandlerTest.java | 2 +- .../IgniteClusterSnapshotRestoreSelfTest.java | 2 +- .../IgniteClusterSnapshotSelfTest.java | 2 +- 11 files changed, 40 insertions(+), 38 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 3cf17258ab2848..c96d969ec71ca8 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3356,7 +3356,7 @@ public void testSnapshotCreateCheckAndRestoreCustomDir() throws Exception { String snpName = "snapshot_30052022"; File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); - assert snpDir.list().length == 0 : "Target directory is not empty: " + Arrays.asList(snpDir.list()); + assert snpDir.list().length == 0 : "Target directory is not empty: " + snpDir; try { Ignite ignite = startGrids(2); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java index 122775bc0a1b4a..8bfcab0c7d2768 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotVerificationTask.java @@ -123,10 +123,10 @@ public static void checkMissedMetadata(Collection clusterMetas /** * @param name Snapshot name. - * @param snpPath Snapshot directory path. + * @param path Snapshot directory path. * @param constId Snapshot metadata file name. * @param groups Cache groups to be restored from the snapshot. May be empty if all cache groups are being restored. * @return Compute job. */ - protected abstract ComputeJob createJob(String name, @Nullable String snpPath, String constId, Collection groups); + protected abstract ComputeJob createJob(String name, @Nullable String path, String constId, Collection groups); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index f98489e7d33162..ffb73fe6510ba3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -1652,14 +1652,17 @@ public static boolean isSnapshotOperation(DiscoveryEvent evt) { } /** - * @param parts Collection of pairs group and appropriate cache partition to be snapshot. * @param rmtNodeId The remote node to connect to. + * @param snpName Snapshot name to request. + * @param rmtSnpPath Snapshot directory path on the remote node. + * @param parts Collection of pairs group and appropriate cache partition to be snapshot. + * @param stopChecker Node stop or prcoess interrupt checker. * @param partHnd Received partition handler. */ public IgniteInternalFuture requestRemoteSnapshotFiles( UUID rmtNodeId, String snpName, - String snpPath, + @Nullable String rmtSnpPath, Map> parts, BooleanSupplier stopChecker, BiConsumer<@Nullable File, @Nullable Throwable> partHnd @@ -1678,7 +1681,7 @@ public IgniteInternalFuture requestRemoteSnapshotFiles( throw new IgniteCheckedException("Snapshot on remote node is not supported: " + rmtNode.id()); RemoteSnapshotFilesRecevier fut = - new RemoteSnapshotFilesRecevier(this, rmtNodeId, snpName, snpPath, parts, stopChecker, partHnd); + new RemoteSnapshotFilesRecevier(this, rmtNodeId, snpName, rmtSnpPath, parts, stopChecker, partHnd); snpRmtMgr.submit(fut); @@ -1710,7 +1713,7 @@ private static String snapshotMetaFileName(String consId) { } /** - * @param snpName Snapshot name. + * @param snpDir The full path to the snapshot files. * @param folderName The node folder name, usually it's the same as the U.maskForFileName(consistentId). * @return Standalone kernal context related to the snapshot. * @throws IgniteCheckedException If fails. @@ -2487,7 +2490,7 @@ private static class RemoteSnapshotFilesRecevier extends GridFutureAdapter * @param snpMgr Ignite snapshot manager. * @param rmtNodeId Remote node to request snapshot from. * @param snpName Snapshot name to request. - * @param rmtSnpPath Snapshot directory path. + * @param rmtSnpPath Snapshot directory path on the remote node. * @param parts Cache group and partitions to request. * @param stopChecker Process interrupt checker. * @param partHnd Partition handler. @@ -2496,7 +2499,7 @@ public RemoteSnapshotFilesRecevier( IgniteSnapshotManager snpMgr, UUID rmtNodeId, String snpName, - String rmtSnpPath, + @Nullable String rmtSnpPath, Map> parts, BooleanSupplier stopChecker, BiConsumer<@Nullable File, @Nullable Throwable> partHnd @@ -3036,8 +3039,6 @@ public RemoteSnapshotSender( assert len > 0 : "Requested partitions has incorrect file length " + "[pair=" + pair + ", cacheDirName=" + cacheDirName + ']'; - System.out.println(">xxx> send " + part); - sndr.send(part, 0, len, transmissionParams(rqId, cacheDirName, pair), TransmissionPolicy.FILE); if (log.isInfoEnabled()) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFilesRequestMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFilesRequestMessage.java index 413d62e596a041..81c9870477b95a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFilesRequestMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFilesRequestMessage.java @@ -62,7 +62,7 @@ public SnapshotFilesRequestMessage() { /** * @param reqId Unique request id. - * @param snpName Snapshot name. + * @param snpName Snapshot name to request. * @param snpPath Snapshot directory path. * @param parts Map of cache group ids and corresponding set of its partition ids to be snapshot. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java index e1cb2570380987..947085ab87f589 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotHandlerRestoreTask.java @@ -49,12 +49,12 @@ public class SnapshotHandlerRestoreTask extends AbstractSnapshotVerificationTask /** {@inheritDoc} */ @Override protected ComputeJob createJob( - String snpName, - @Nullable String snpPath, + String name, + @Nullable String path, String constId, Collection groups ) { - return new SnapshotHandlerRestoreJob(snpName, snpPath, constId, groups); + return new SnapshotHandlerRestoreJob(name, path, constId, groups); } /** {@inheritDoc} */ @@ -129,9 +129,9 @@ private static class SnapshotHandlerRestoreJob extends ComputeJobAdapter { */ public SnapshotHandlerRestoreJob(String snpName, @Nullable String snpPath, String consistentId, Collection grps) { this.snpName = snpName; + this.snpPath = snpPath; this.consistentId = consistentId; this.grps = grps; - this.snpPath = snpPath; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java index 5274d9b5f605a8..05f5856b903868 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java @@ -50,13 +50,9 @@ public class SnapshotPartitionsVerifyTask extends AbstractSnapshotVerificationTa /** Serial version uid. */ private static final long serialVersionUID = 0L; - /** Ignite instance. */ - @IgniteInstanceResource - private IgniteEx ignite; - /** {@inheritDoc} */ - @Override protected ComputeJob createJob(String name, String snpPath, String constId, Collection groups) { - return new VisorVerifySnapshotPartitionsJob(name, snpPath, constId, groups); + @Override protected ComputeJob createJob(String name, String path, String constId, Collection groups) { + return new VisorVerifySnapshotPartitionsJob(name, path, constId, groups); } /** {@inheritDoc} */ @@ -80,22 +76,27 @@ private static class VisorVerifySnapshotPartitionsJob extends ComputeJobAdapter /** Snapshot name to validate. */ private final String snpName; + /** Snapshot directory path. */ + private final String snpPath; + /** Consistent snapshot metadata file name. */ private final String consId; /** Set of cache groups to be checked in the snapshot or {@code empty} to check everything. */ private final Collection rqGrps; - /** Snapshot directory path. */ - private final String snpPath; - /** * @param snpName Snapshot name to validate. * @param consId Consistent snapshot metadata file name. * @param rqGrps Set of cache groups to be checked in the snapshot or {@code empty} to check everything. * @param snpPath Snapshot directory path. */ - public VisorVerifySnapshotPartitionsJob(String snpName, @Nullable String snpPath, String consId, Collection rqGrps) { + public VisorVerifySnapshotPartitionsJob( + String snpName, + @Nullable String snpPath, + String consId, + Collection rqGrps + ) { this.snpName = snpName; this.consId = consId; this.rqGrps = rqGrps; @@ -132,13 +133,14 @@ public VisorVerifySnapshotPartitionsJob(String snpName, @Nullable String snpPath return false; VisorVerifySnapshotPartitionsJob job = (VisorVerifySnapshotPartitionsJob)o; - // todo rqGrps snpPath - return snpName.equals(job.snpName) && consId.equals(job.consId); + + return snpName.equals(job.snpName) && consId.equals(job.consId) && + Objects.equals(rqGrps, job.rqGrps) && Objects.equals(snpPath, job.snpPath); } /** {@inheritDoc} */ @Override public int hashCode() { - return Objects.hash(snpName, consId); + return Objects.hash(snpName, consId, rqGrps, snpPath); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java index fa974d4b5666ad..6ccf2000a5520f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java @@ -39,14 +39,14 @@ /** */ public class SnapshotResponseRemoteFutureTask extends AbstractSnapshotFutureTask { - /** */ + /** Snapshot directory path. */ private final String snpPath; /** * @param cctx Shared context. * @param srcNodeId Node id which cause snapshot task creation. * @param snpName Unique identifier of snapshot process. - * @param snpPath + * @param snpPath Snapshot directory path. * @param tmpWorkDir Working directory for intermediate snapshot results. * @param ioFactory Factory to working with snapshot files. * @param snpSndr Factory which produces snapshot receiver instance. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 006b13deb5355d..1b0a176c932ef7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -827,9 +827,6 @@ private IgniteInternalFuture preload(UUID reqId) { if (ctx.isStopping()) throw new NodeStoppingException("Node is stopping: " + ctx.localNodeId()); - File snpDir = opCtx0.snpPath != null ? new File(opCtx0.snpPath, opCtx0.snpName) : - ctx.cache().context().snapshotMgr().snapshotLocalDir(opCtx0.snpName); - Set allMetas = opCtx0.metasPerNode.values().stream().flatMap(List::stream).collect(Collectors.toSet()); @@ -848,6 +845,8 @@ private IgniteInternalFuture preload(UUID reqId) { ", caches=" + F.transform(opCtx0.dirs, FilePageStoreManager::cacheGroupName) + ']'); } + File snpDir = snpMgr.snapshotLocalDir(opCtx0.snpName, opCtx0.snpPath); + CompletableFuture metaFut = ctx.localNodeId().equals(opCtx0.opNodeId) ? CompletableFuture.runAsync( () -> { @@ -996,7 +995,7 @@ private IgniteInternalFuture preload(UUID reqId) { ctx.cache().context().snapshotMgr() .requestRemoteSnapshotFiles(m.getKey(), opCtx0.snpName, - opCtx0.snpPath == null ? null : opCtx0.snpPath.toString(), + opCtx0.snpPath, m.getValue(), opCtx0.stopChecker, (snpFile, t) -> { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java index 99bb718433c366..a3b4600b5b032d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java @@ -351,7 +351,7 @@ public void testCrdChangeDuringHandlerCompleteOnSnapshotCreate() throws Exceptio } /** - * Test ensures that the location of the snapshot is valid. + * Test ensures that the snapshot path is set correctly in the handler context. * * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index 85b56029415381..cee3f61ece1590 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -127,7 +127,7 @@ public void testRestoreWithEmptyPartitions() throws Exception { public void testClusterSnapshotRestoreFromCustomDir() throws Exception { File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); - assert snpDir.list().length == 0 : "Target directory is not empty: " + Arrays.asList(snpDir.list()); + assert snpDir.list().length == 0 : "Target directory is not empty: " + snpDir; try { IgniteEx ignite = startGrids(2); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java index 98ffb8be1e062d..26d375c3305a35 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java @@ -722,7 +722,7 @@ public void testClusterSnapshotWithExplicitPath() throws Exception { private void doTestClusterSnapshotWithExplicitPath(boolean cfgPath) throws Exception { File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); - assert snpDir.list().length == 0 : "Target directory is not empty: " + Arrays.asList(snpDir.list()); + assert snpDir.list().length == 0 : "Target directory is not empty: " + snpDir; try { IgniteEx ignite = null; From 69a2881fefc768b6691a5dbf734c6ec3254b7ac7 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Wed, 1 Jun 2022 16:14:19 +0300 Subject: [PATCH 14/19] IGNITE-15067 Fix usage. --- .../commandline/snapshot/SnapshotCheckCommand.java | 2 +- .../snapshot/SnapshotCreateCommand.java | 4 ++-- .../snapshot/SnapshotCreateCommandOption.java | 2 +- .../snapshot/SnapshotRestoreCommand.java | 6 +++--- ...ridCommandHandlerClusterByClassTest_help.output | 14 +++++++------- ...andHandlerClusterByClassWithSSLTest_help.output | 14 +++++++------- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java index b3003eff78b33c..b5d5d76b79d29a 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java @@ -71,7 +71,7 @@ else if (option == SOURCE) { @Override public void printUsage(Logger log) { Map params = new LinkedHashMap<>(generalUsageOptions()); - params.put(SOURCE.optionName(), SOURCE.description()); + params.put(SOURCE.argName() + " " + SOURCE.optionName(), SOURCE.description()); usage(log, "Check snapshot:", SNAPSHOT, params, name(), SNAPSHOT_NAME_ARG, optional(SOURCE.argName(), SOURCE.optionName())); diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java index 0cd6eb16c17294..b70155ebc18265 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java @@ -75,8 +75,8 @@ else if (option == SYNC) @Override public void printUsage(Logger log) { Map params = new LinkedHashMap<>(generalUsageOptions()); - params.put(DESTINATION.optionName(), DESTINATION.description()); - params.put(SYNC.optionName(), SYNC.description()); + params.put(DESTINATION.argName() + " " + DESTINATION.optionName(), DESTINATION.description()); + params.put(SYNC.argName(), SYNC.description()); usage(log, "Create cluster snapshot:", SNAPSHOT, params, name(), SNAPSHOT_NAME_ARG, optional(DESTINATION.argName(), DESTINATION.optionName()), optional(SYNC.argName())); diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java index f5b6db20d44ab0..4bf9aefc687c0b 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java @@ -24,7 +24,7 @@ */ public enum SnapshotCreateCommandOption implements CommandArg { /** Synchronous execution flag. */ - SYNC("--sync", "sync", "Run the operation synchronously, the command will wait for the entire operation to complete. " + + SYNC("--sync", null, "Run the operation synchronously, the command will wait for the entire operation to complete. " + "Otherwise, it will be performed in the background, and the command will immediately return control."), /** Snapshot directory path. */ diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java index 4f946523ba76e1..06a071a6343b98 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java @@ -106,9 +106,9 @@ else if (option == SYNC) Map params = generalUsageOptions(); Map startParams = new LinkedHashMap<>(params); - startParams.put(GROUPS.optionName(), GROUPS.description()); - startParams.put(SOURCE.optionName(), SOURCE.description()); - startParams.put(SYNC.optionName(), SYNC.description()); + startParams.put(GROUPS.argName() + " " + GROUPS.optionName(), GROUPS.description()); + startParams.put(SOURCE.argName() + " " + SOURCE.optionName(), SOURCE.description()); + startParams.put(SYNC.argName(), SYNC.description()); usage(log, "Restore snapshot:", SNAPSHOT, startParams, RESTORE.toString(), SNAPSHOT_NAME_ARG, "--start", optional(GROUPS.argName(), GROUPS.optionName()), optional(SOURCE.argName(), SOURCE.optionName()), optional(SYNC.argName())); diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output index 07271c9fd36724..a05d70951cc9ad 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output @@ -144,8 +144,8 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. - path - Path to the directory where the snapshot will be saved. If not specified, the default snapshot directory will be used. - sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. + --dest path - Path to the directory where the snapshot will be saved. If not specified, the default snapshot directory will be used. + --sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Cancel running snapshot: control.(sh|bat) --snapshot cancel snapshot_name @@ -158,16 +158,16 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. - path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. + --src path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. Restore snapshot: control.(sh|bat) --snapshot restore snapshot_name --start [--groups group1,...groupN] [--src path] [--sync] Parameters: - snapshot_name - Snapshot name. - group1,...groupN - Cache group names. - path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. - sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. + snapshot_name - Snapshot name. + --groups group1,...groupN - Cache group names. + --src path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. + --sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Snapshot restore operation status: control.(sh|bat) --snapshot restore snapshot_name --status diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output index 07271c9fd36724..a05d70951cc9ad 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output @@ -144,8 +144,8 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. - path - Path to the directory where the snapshot will be saved. If not specified, the default snapshot directory will be used. - sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. + --dest path - Path to the directory where the snapshot will be saved. If not specified, the default snapshot directory will be used. + --sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Cancel running snapshot: control.(sh|bat) --snapshot cancel snapshot_name @@ -158,16 +158,16 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. - path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. + --src path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. Restore snapshot: control.(sh|bat) --snapshot restore snapshot_name --start [--groups group1,...groupN] [--src path] [--sync] Parameters: - snapshot_name - Snapshot name. - group1,...groupN - Cache group names. - path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. - sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. + snapshot_name - Snapshot name. + --groups group1,...groupN - Cache group names. + --src path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. + --sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Snapshot restore operation status: control.(sh|bat) --snapshot restore snapshot_name --status From 1ffe57d8783ec80ce0631f451c06b3f1c95cb2d6 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Wed, 1 Jun 2022 16:17:08 +0300 Subject: [PATCH 15/19] IGNITE-15067 Assert replaced with junit assertion in test. --- .../java/org/apache/ignite/util/GridCommandHandlerTest.java | 2 +- .../snapshot/IgniteClusterSnapshotRestoreSelfTest.java | 2 +- .../persistence/snapshot/IgniteClusterSnapshotSelfTest.java | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index c96d969ec71ca8..2b791eea0493bc 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3356,7 +3356,7 @@ public void testSnapshotCreateCheckAndRestoreCustomDir() throws Exception { String snpName = "snapshot_30052022"; File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); - assert snpDir.list().length == 0 : "Target directory is not empty: " + snpDir; + assertTrue("Target directory is not empty: " + snpDir, F.isEmpty(snpDir.list())); try { Ignite ignite = startGrids(2); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index cee3f61ece1590..8cb9049fd497c3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -127,7 +127,7 @@ public void testRestoreWithEmptyPartitions() throws Exception { public void testClusterSnapshotRestoreFromCustomDir() throws Exception { File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); - assert snpDir.list().length == 0 : "Target directory is not empty: " + snpDir; + assertTrue("Target directory is not empty: " + snpDir, F.isEmpty(snpDir.list())); try { IgniteEx ignite = startGrids(2); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java index 26d375c3305a35..3342619fe3856c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java @@ -72,6 +72,7 @@ import org.apache.ignite.internal.util.distributed.DistributedProcess; import org.apache.ignite.internal.util.distributed.FullMessage; import org.apache.ignite.internal.util.distributed.SingleNodeMessage; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.CU; @@ -722,7 +723,7 @@ public void testClusterSnapshotWithExplicitPath() throws Exception { private void doTestClusterSnapshotWithExplicitPath(boolean cfgPath) throws Exception { File snpDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "ex_snapshots", true); - assert snpDir.list().length == 0 : "Target directory is not empty: " + snpDir; + assertTrue("Target directory is not empty: " + snpDir, F.isEmpty(snpDir.list())); try { IgniteEx ignite = null; From 4d23d81c70f86608ff318d975ac50dbd961284a8 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Wed, 1 Jun 2022 17:13:28 +0300 Subject: [PATCH 16/19] IGNITE-15067 Review note. --- .../snapshot/SnapshotCheckCommand.java | 4 +- .../snapshot/SnapshotCheckCommandOption.java | 32 ++++++++-------- .../snapshot/SnapshotCreateCommand.java | 4 +- .../snapshot/SnapshotCreateCommandOption.java | 33 +++++++++-------- .../snapshot/SnapshotRestoreCommand.java | 6 +-- .../SnapshotRestoreCommandOption.java | 37 ++++++++++--------- 6 files changed, 59 insertions(+), 57 deletions(-) diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java index b5d5d76b79d29a..ab858936c89243 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java @@ -71,10 +71,10 @@ else if (option == SOURCE) { @Override public void printUsage(Logger log) { Map params = new LinkedHashMap<>(generalUsageOptions()); - params.put(SOURCE.argName() + " " + SOURCE.optionName(), SOURCE.description()); + params.put(SOURCE.argName() + " " + SOURCE.arg(), SOURCE.description()); usage(log, "Check snapshot:", SNAPSHOT, params, name(), SNAPSHOT_NAME_ARG, - optional(SOURCE.argName(), SOURCE.optionName())); + optional(SOURCE.argName(), SOURCE.arg())); } /** {@inheritDoc} */ diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java index 52c463d66f0a72..761c76fae6256f 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java @@ -27,37 +27,37 @@ public enum SnapshotCheckCommandOption implements CommandArg { SOURCE("--src", "path", "Path to the directory where the snapshot files are located. If not specified, " + "the default snapshot directory will be used."); - /** Option name. */ - private final String argName; + /** Name. */ + private final String name; - /** Option name. */ - private final String optionName; + /** Argument. */ + private final String arg; - /** Option description. */ + /** Description. */ private final String desc; /** - * @param argName Argument name. - * @param optionName Option name. - * @param desc Option description. + * @param name Name. + * @param arg Argument. + * @param desc Description. */ - SnapshotCheckCommandOption(String argName, String optionName, String desc) { - this.argName = argName; - this.optionName = optionName; + SnapshotCheckCommandOption(String name, String arg, String desc) { + this.name = name; + this.arg = arg; this.desc = desc; } /** {@inheritDoc} */ @Override public String argName() { - return argName; + return name; } - /** @return Option name. */ - public String optionName() { - return optionName; + /** @return Argument. */ + public String arg() { + return arg; } - /** @return Option description. */ + /** @return Description. */ public String description() { return desc; } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java index b70155ebc18265..ad621601266287 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java @@ -75,10 +75,10 @@ else if (option == SYNC) @Override public void printUsage(Logger log) { Map params = new LinkedHashMap<>(generalUsageOptions()); - params.put(DESTINATION.argName() + " " + DESTINATION.optionName(), DESTINATION.description()); + params.put(DESTINATION.argName() + " " + DESTINATION.arg(), DESTINATION.description()); params.put(SYNC.argName(), SYNC.description()); usage(log, "Create cluster snapshot:", SNAPSHOT, params, name(), SNAPSHOT_NAME_ARG, - optional(DESTINATION.argName(), DESTINATION.optionName()), optional(SYNC.argName())); + optional(DESTINATION.argName(), DESTINATION.arg()), optional(SYNC.argName())); } } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java index 4bf9aefc687c0b..05875ddf4d7ab1 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.commandline.snapshot; import org.apache.ignite.internal.commandline.argument.CommandArg; +import org.jetbrains.annotations.Nullable; /** * Snapshot create command options. @@ -31,37 +32,37 @@ public enum SnapshotCreateCommandOption implements CommandArg { DESTINATION("--dest", "path", "Path to the directory where the snapshot will be saved. If not specified, " + "the default snapshot directory will be used."); - /** Option name. */ - private final String argName; + /** Name. */ + private final String name; - /** Option name. */ - private final String optionName; + /** Argument. */ + private final String arg; - /** Option description. */ + /** Description. */ private final String desc; /** - * @param argName Argument name. - * @param optionName Option name. - * @param desc Option description. + * @param name Name. + * @param arg Argument. + * @param desc Description. */ - SnapshotCreateCommandOption(String argName, String optionName, String desc) { - this.argName = argName; - this.optionName = optionName; + SnapshotCreateCommandOption(String name, @Nullable String arg, String desc) { + this.name = name; + this.arg = arg; this.desc = desc; } /** {@inheritDoc} */ @Override public String argName() { - return argName; + return name; } - /** @return Option name. */ - public String optionName() { - return optionName; + /** @return Argument. */ + public String arg() { + return arg; } - /** @return Option description. */ + /** @return Description. */ public String description() { return desc; } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java index 06a071a6343b98..c5869dbf818862 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java @@ -106,12 +106,12 @@ else if (option == SYNC) Map params = generalUsageOptions(); Map startParams = new LinkedHashMap<>(params); - startParams.put(GROUPS.argName() + " " + GROUPS.optionName(), GROUPS.description()); - startParams.put(SOURCE.argName() + " " + SOURCE.optionName(), SOURCE.description()); + startParams.put(GROUPS.argName() + " " + GROUPS.arg(), GROUPS.description()); + startParams.put(SOURCE.argName() + " " + SOURCE.arg(), SOURCE.description()); startParams.put(SYNC.argName(), SYNC.description()); usage(log, "Restore snapshot:", SNAPSHOT, startParams, RESTORE.toString(), SNAPSHOT_NAME_ARG, "--start", - optional(GROUPS.argName(), GROUPS.optionName()), optional(SOURCE.argName(), SOURCE.optionName()), optional(SYNC.argName())); + optional(GROUPS.argName(), GROUPS.arg()), optional(SOURCE.argName(), SOURCE.arg()), optional(SYNC.argName())); usage(log, "Snapshot restore operation status:", SNAPSHOT, params, RESTORE.toString(), SNAPSHOT_NAME_ARG, "--status"); usage(log, "Cancel snapshot restore operation:", SNAPSHOT, params, RESTORE.toString(), SNAPSHOT_NAME_ARG, "--cancel"); } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java index a0532ac2519aff..eb66ccfde01bb9 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommandOption.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.commandline.snapshot; import org.apache.ignite.internal.commandline.argument.CommandArg; +import org.jetbrains.annotations.Nullable; /** * Snapshot restore command options. @@ -27,44 +28,44 @@ public enum SnapshotRestoreCommandOption implements CommandArg { GROUPS("--groups", "group1,...groupN", "Cache group names."), /** Snapshot directory location. */ - SOURCE(SnapshotCheckCommandOption.SOURCE.argName(), SnapshotCheckCommandOption.SOURCE.optionName(), + SOURCE(SnapshotCheckCommandOption.SOURCE.argName(), SnapshotCheckCommandOption.SOURCE.arg(), SnapshotCheckCommandOption.SOURCE.description()), /** Synchronous execution flag. */ - SYNC(SnapshotCreateCommandOption.SYNC.argName(), SnapshotCreateCommandOption.SYNC.optionName(), + SYNC(SnapshotCreateCommandOption.SYNC.argName(), SnapshotCreateCommandOption.SYNC.arg(), SnapshotCreateCommandOption.SYNC.description()); - /** Argument name. */ - private final String argName; + /** Name. */ + private final String name; - /** Option name. */ - private final String optionName; + /** Argument. */ + private final String arg; - /** Option description. */ + /** Description. */ private final String desc; /** - * @param argName Argument name. - * @param optionName Option name. - * @param desc Option description. + * @param name Name. + * @param arg Argument. + * @param desc Description. */ - SnapshotRestoreCommandOption(String argName, String optionName, String desc) { - this.argName = argName; - this.optionName = optionName; + SnapshotRestoreCommandOption(String name, @Nullable String arg, String desc) { + this.name = name; + this.arg = arg; this.desc = desc; } /** {@inheritDoc} */ @Override public String argName() { - return argName; + return name; } - /** @return Option name. */ - public String optionName() { - return optionName; + /** @return Argument. */ + public String arg() { + return arg; } - /** @return Option description. */ + /** @return Description. */ public String description() { return desc; } From 1ad052875563bf4e72b31aeae713e52c3c764415 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Wed, 1 Jun 2022 20:33:02 +0300 Subject: [PATCH 17/19] IGNITE-15067 Forbid option duplication. --- .../commandline/snapshot/SnapshotCheckCommand.java | 3 +++ .../commandline/snapshot/SnapshotCreateCommand.java | 9 ++++++++- .../commandline/snapshot/SnapshotRestoreCommand.java | 12 +++++++++++- .../apache/ignite/util/GridCommandHandlerTest.java | 7 +++++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java index ab858936c89243..d298c87a6f6370 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommand.java @@ -55,6 +55,9 @@ protected SnapshotCheckCommand() { "Possible options: " + F.concat(F.asList(SnapshotCheckCommandOption.values()), ", ") + '.'); } else if (option == SOURCE) { + if (snpPath != null) + throw new IllegalArgumentException(SOURCE.argName() + " arg specified twice."); + String errMsg = "Expected path to the snapshot directory."; if (CommandArgIterator.isCommandOrOption(argIter.peekNextArg())) diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java index ad621601266287..7582bd3fcfbb5e 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommand.java @@ -56,6 +56,9 @@ protected SnapshotCreateCommand() { "Possible options: " + F.concat(F.asList(SnapshotCreateCommandOption.values()), ", ") + '.'); } else if (option == DESTINATION) { + if (snpPath != null) + throw new IllegalArgumentException(DESTINATION.argName() + " arg specified twice."); + String errMsg = "Expected path to the snapshot directory."; if (CommandArgIterator.isCommandOrOption(argIter.peekNextArg())) @@ -63,8 +66,12 @@ else if (option == DESTINATION) { snpPath = argIter.nextArg(errMsg); } - else if (option == SYNC) + else if (option == SYNC) { + if (sync) + throw new IllegalArgumentException(SYNC.argName() + " arg specified twice."); + sync = true; + } } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java index c5869dbf818862..0fa21259a54090 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotRestoreCommand.java @@ -79,6 +79,9 @@ protected SnapshotRestoreCommand() { "Possible options: " + F.concat(F.asList(SnapshotRestoreCommandOption.values()), ", ") + '.'); } else if (option == GROUPS) { + if (grpNames != null) + throw new IllegalArgumentException(GROUPS.argName() + " arg specified twice."); + String argDesc = "a comma-separated list of cache group names."; grpNames = argIter.nextStringSet(argDesc); @@ -87,6 +90,9 @@ else if (option == GROUPS) { throw new IllegalArgumentException("Expected " + argDesc); } else if (option == SOURCE) { + if (snpPath != null) + throw new IllegalArgumentException(SOURCE.argName() + " arg specified twice."); + String errMsg = "Expected path to the snapshot directory."; if (CommandArgIterator.isCommandOrOption(argIter.peekNextArg())) @@ -94,8 +100,12 @@ else if (option == SOURCE) { snpPath = argIter.nextArg(errMsg); } - else if (option == SYNC) + else if (option == SYNC) { + if (sync) + throw new IllegalArgumentException(SYNC.argName() + " arg specified twice."); + sync = true; + } } cmdArg = new VisorSnapshotRestoreTaskArg(snpName, snpPath, sync, restoreAction, grpNames); diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 2b791eea0493bc..1669e2251e6451 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3367,6 +3367,9 @@ public void testSnapshotCreateCheckAndRestoreCustomDir() throws Exception { injectTestSystemOut(); CommandHandler h = new CommandHandler(); + assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "create", snpName, "--dest", "A", "--dest", "B")); + assertContains(log, testOut.toString(), "--dest arg specified twice."); + assertEquals(EXIT_CODE_OK, execute(h, "--snapshot", "create", snpName, "--sync", "--dest", snpDir.getAbsolutePath())); @@ -3375,6 +3378,10 @@ public void testSnapshotCreateCheckAndRestoreCustomDir() throws Exception { assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute(h, "--snapshot", "restore", snpName, "--start", "--sync")); assertContains(log, testOut.toString(), "Snapshot does not exists [snapshot=" + snpName); + assertEquals(EXIT_CODE_INVALID_ARGUMENTS, + execute(h, "--snapshot", "restore", snpName, "--start", "--src", "A", "--src", "B")); + assertContains(log, testOut.toString(), "--src arg specified twice."); + // The check command simply prints the results of the check, it always ends with a zero exit code. assertEquals(EXIT_CODE_OK, execute(h, "--snapshot", "check", snpName)); assertContains(log, testOut.toString(), "Snapshot does not exists [snapshot=" + snpName); From a380eed1263cc83e39a59e372f1e5dd5dc4b9056 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Thu, 2 Jun 2022 13:20:30 +0300 Subject: [PATCH 18/19] IGNITE-15067 Review note - default configured. --- .../commandline/snapshot/SnapshotCheckCommandOption.java | 2 +- .../commandline/snapshot/SnapshotCreateCommandOption.java | 2 +- .../GridCommandHandlerClusterByClassTest_help.output | 6 +++--- .../GridCommandHandlerClusterByClassWithSSLTest_help.output | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java index 761c76fae6256f..c3847ea4e49923 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCheckCommandOption.java @@ -25,7 +25,7 @@ public enum SnapshotCheckCommandOption implements CommandArg { /** Snapshot directory location. */ SOURCE("--src", "path", "Path to the directory where the snapshot files are located. If not specified, " + - "the default snapshot directory will be used."); + "the default configured snapshot directory will be used."); /** Name. */ private final String name; diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java index 05875ddf4d7ab1..a6d216d178185b 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCreateCommandOption.java @@ -30,7 +30,7 @@ public enum SnapshotCreateCommandOption implements CommandArg { /** Snapshot directory path. */ DESTINATION("--dest", "path", "Path to the directory where the snapshot will be saved. If not specified, " + - "the default snapshot directory will be used."); + "the default configured snapshot directory will be used."); /** Name. */ private final String name; diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output index a05d70951cc9ad..39024a5c748fbf 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output @@ -144,7 +144,7 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. - --dest path - Path to the directory where the snapshot will be saved. If not specified, the default snapshot directory will be used. + --dest path - Path to the directory where the snapshot will be saved. If not specified, the default configured snapshot directory will be used. --sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Cancel running snapshot: @@ -158,7 +158,7 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. - --src path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. + --src path - Path to the directory where the snapshot files are located. If not specified, the default configured snapshot directory will be used. Restore snapshot: control.(sh|bat) --snapshot restore snapshot_name --start [--groups group1,...groupN] [--src path] [--sync] @@ -166,7 +166,7 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. --groups group1,...groupN - Cache group names. - --src path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. + --src path - Path to the directory where the snapshot files are located. If not specified, the default configured snapshot directory will be used. --sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Snapshot restore operation status: diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output index a05d70951cc9ad..39024a5c748fbf 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output @@ -144,7 +144,7 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. - --dest path - Path to the directory where the snapshot will be saved. If not specified, the default snapshot directory will be used. + --dest path - Path to the directory where the snapshot will be saved. If not specified, the default configured snapshot directory will be used. --sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Cancel running snapshot: @@ -158,7 +158,7 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. - --src path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. + --src path - Path to the directory where the snapshot files are located. If not specified, the default configured snapshot directory will be used. Restore snapshot: control.(sh|bat) --snapshot restore snapshot_name --start [--groups group1,...groupN] [--src path] [--sync] @@ -166,7 +166,7 @@ This utility can do the following commands: Parameters: snapshot_name - Snapshot name. --groups group1,...groupN - Cache group names. - --src path - Path to the directory where the snapshot files are located. If not specified, the default snapshot directory will be used. + --src path - Path to the directory where the snapshot files are located. If not specified, the default configured snapshot directory will be used. --sync - Run the operation synchronously, the command will wait for the entire operation to complete. Otherwise, it will be performed in the background, and the command will immediately return control. Snapshot restore operation status: From 05577c610cd3262fd83f24397742838f97a66386 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Fri, 3 Jun 2022 15:28:34 +0300 Subject: [PATCH 19/19] IGNITE-15067 (minor) Store absolute path. --- .../cache/persistence/snapshot/IgniteSnapshotManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index ffb73fe6510ba3..7647eb4b736ba5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -3138,7 +3138,7 @@ public LocalSnapshotSender(String snpName, @Nullable String snpPath) { assert metaStorage != null && metaStorage.read(SNP_RUNNING_DIR_KEY) == null : "The previous snapshot hasn't been completed correctly"; - metaStorage.write(SNP_RUNNING_DIR_KEY, snpLocDir.toString()); + metaStorage.write(SNP_RUNNING_DIR_KEY, snpLocDir.getAbsolutePath()); U.ensureDirectory(dbDir, "snapshot work directory for a local snapshot sender", log); }