+ * This is an emergency recovery tool for debugging when a Cassandra instance cannot + * start due to TCM issues. It reads the local_metadata_log and metadata_snapshots + * tables from the system keyspace to reconstruct and display the cluster metadata state. + *
+ * Usage: + *
+ * # Binary dump (default) + * tcmdump dump --data-dir /path/to/data + * + * # toString output for debugging + * tcmdump dump --data-dir /path/to/data --to-string + * + * # Dump log entries + * tcmdump dump --data-dir /path/to/data --dump-log --from-epoch 1 --to-epoch 50 + * + * # Dump distributed log (CMS nodes) + * tcmdump dump --data-dir /path/to/data --dump-distributed-log + *+ */ +@Command(name = "tcmdump", +mixinStandardHelpOptions = true, +description = "Dump Transactional Cluster Metadata from local SSTables", +subcommands = { TCMDump.DumpMetadata.class }) +public class TCMDump implements Runnable +{ + private static final Output output = Output.CONSOLE; + + public static void main(String... args) + { + Util.initDatabaseDescriptor(); + + CommandLine cli = new CommandLine(TCMDump.class).setExecutionExceptionHandler((ex, cmd, parseResult) -> { + err(ex); + return 2; + }); + int status = cli.execute(args); + System.exit(status); + } + + protected static void err(Throwable e) + { + output.err.println("error: " + e.getMessage()); + output.err.println("-- StackTrace --"); + output.err.println(getStackTraceAsString(e)); + } + + @Override + public void run() + { + CommandLine.usage(this, output.out); + } + + @Command(name = "dump", description = "Dump cluster metadata from SSTables") + public static class DumpMetadata implements Runnable + { + @Option(names = { "-d", "--data-dir" }, description = "Data directory containing system keyspace") + public String dataDir; + + @Option(names = { "-s", "--sstables" }, description = "Path to SSTable directory for metadata tables (can be specified multiple times for log and snapshot tables)", arity = "1..*") + public List
+ * It detects gaps in the epoch sequence and logs warnings instead of throwing exceptions, allowing the tool to
+ * still output all available epochs.
+ *
+ * @param storage the storage to read entries from
+ * @param snapshotManager the snapshot manager for base state
+ * @param targetEpoch optional target epoch to filter to (null for all epochs)
+ * @param out the output to write warnings to
+ * @return the LogState with all available entries
+ */
+ @VisibleForTesting
+ static LogState getLogState(SystemKeyspaceStorage storage,
+ MetadataSnapshots snapshotManager,
+ Long targetEpoch,
+ Output out)
+ {
+ ClusterMetadata base = snapshotManager.getLatestSnapshot();
+ Epoch baseEpoch = base == null ? Epoch.EMPTY : base.epoch;
+ Epoch endEpoch = targetEpoch != null ? Epoch.create(targetEpoch) : Epoch.create(Long.MAX_VALUE);
+
+ try
+ {
+ LogReader.EntryHolder entryHolder = storage.getEntries(baseEpoch, endEpoch);
+ ImmutableList.Builder
+ * These tests write entries directly to the system keyspace storage and then
+ * call TCMDump.DumpMetadata.getLogState() directly to verify gap detection behavior.
+ */
+public class TCMDumpIntegrationTest extends OfflineToolUtils
+{
+ private SystemKeyspaceStorage storage;
+ private MetadataSnapshots snapshotManager;
+
+ @BeforeClass
+ public static void setupClass() throws IOException
+ {
+ DatabaseDescriptor.daemonInitialization();
+ StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
+ ServerTestUtils.prepareServerNoRegister();
+ CommitLog.instance.start();
+ }
+
+ @Before
+ public void setup()
+ {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(SYSTEM_KEYSPACE_NAME, METADATA_LOG);
+ if (cfs != null)
+ cfs.truncateBlockingWithoutSnapshot();
+
+ snapshotManager = new MetadataSnapshots.SystemKeyspaceMetadataSnapshots();
+ storage = new SystemKeyspaceStorage(() -> snapshotManager);
+ }
+
+ private Entry entry(long epoch)
+ {
+ return new Entry(new Entry.Id(epoch),
+ Epoch.create(epoch),
+ CustomTransformation.make((int) epoch));
+ }
+
+ @Test
+ public void testGapDetectionInEpochs()
+ {
+ // Write entries with gap at epoch 3
+ storage.append(entry(1));
+ storage.append(entry(2));
+ storage.append(entry(4)); // Gap: skipping 3
+ storage.append(entry(5));
+
+ TestOutput testOutput = new TestOutput();
+ LogState logState = TCMDump.DumpMetadata.getLogState(storage, snapshotManager, null, testOutput.getOutput());
+ String stderr = testOutput.getStderr();
+
+ // Verify gap is detected and reported
+ assertThat(stderr).contains("Gap detected");
+ assertThat(stderr).contains("expected epoch 3 but found 4");
+
+ // All epochs should still be in the log state
+ assertThat(logState.entries).hasSize(4);
+ assertThat(logState.entries.get(0).epoch.getEpoch()).isEqualTo(1);
+ assertThat(logState.entries.get(1).epoch.getEpoch()).isEqualTo(2);
+ assertThat(logState.entries.get(2).epoch.getEpoch()).isEqualTo(4);
+ assertThat(logState.entries.get(3).epoch.getEpoch()).isEqualTo(5);
+ }
+
+ @Test
+ public void testMultipleGapsDetection()
+ {
+ // Write entries with multiple gaps: missing 2, 4, 6, 7
+ storage.append(entry(1));
+ storage.append(entry(3)); // Gap: skipping 2
+ storage.append(entry(5)); // Gap: skipping 4
+ storage.append(entry(8)); // Gap: skipping 6, 7
+
+ TestOutput testOutput = new TestOutput();
+ LogState logState = TCMDump.DumpMetadata.getLogState(storage, snapshotManager, null, testOutput.getOutput());
+ String stderr = testOutput.getStderr();
+
+ // Verify multiple gaps are detected
+ assertThat(stderr).contains("Gap detected");
+ assertThat(stderr).contains("expected epoch 2 but found 3");
+ assertThat(stderr).contains("expected epoch 4 but found 5");
+ assertThat(stderr).contains("expected epoch 6 but found 8");
+
+ // All available epochs should still be in the log state
+ assertThat(logState.entries).hasSize(4);
+ assertThat(logState.entries.get(0).epoch.getEpoch()).isEqualTo(1);
+ assertThat(logState.entries.get(1).epoch.getEpoch()).isEqualTo(3);
+ assertThat(logState.entries.get(2).epoch.getEpoch()).isEqualTo(5);
+ assertThat(logState.entries.get(3).epoch.getEpoch()).isEqualTo(8);
+ }
+
+ @Test
+ public void testNoGapsNoWarnings()
+ {
+ // No gaps
+ storage.append(entry(1));
+ storage.append(entry(2));
+ storage.append(entry(3));
+ storage.append(entry(4));
+ storage.append(entry(5));
+
+ TestOutput testOutput = new TestOutput();
+ LogState logState = TCMDump.DumpMetadata.getLogState(storage, snapshotManager, null, testOutput.getOutput());
+ String stderr = testOutput.getStderr();
+
+ // Gap warnings should not appear
+ assertThat(stderr).doesNotContain("Gap detected");
+ assertThat(stderr).doesNotContain("WARNING");
+
+ // All entries should be in the log state
+ assertThat(logState.entries).hasSize(5);
+ for (int i = 0; i < 5; i++)
+ {
+ assertThat(logState.entries.get(i).epoch.getEpoch()).isEqualTo(i + 1);
+ }
+ }
+
+ @Test
+ public void testEmptyLogReturnsEmptyState()
+ {
+ // Don't write any entries - log is empty
+ TestOutput testOutput = new TestOutput();
+ LogState logState = TCMDump.DumpMetadata.getLogState(storage, snapshotManager, null, testOutput.getOutput());
+
+ // Should return empty log state
+ assertThat(logState.isEmpty()).isTrue();
+ assertThat(logState.entries).isEmpty();
+ }
+
+ @Test
+ public void testSingleEntryNoGap()
+ {
+ // Single entry at epoch 1
+ storage.append(entry(1));
+
+ TestOutput testOutput = new TestOutput();
+ LogState logState = TCMDump.DumpMetadata.getLogState(storage, snapshotManager, null, testOutput.getOutput());
+ String stderr = testOutput.getStderr();
+
+ // No gap warnings
+ assertThat(stderr).doesNotContain("Gap detected");
+
+ // Single entry should be present
+ assertThat(logState.entries).hasSize(1);
+ assertThat(logState.entries.get(0).epoch.getEpoch()).isEqualTo(1);
+ }
+
+ @Test
+ public void testGapAtBeginning()
+ {
+ // Start with epoch 3 instead of 1 - gap at the beginning
+ storage.append(entry(3));
+ storage.append(entry(4));
+ storage.append(entry(5));
+
+ TestOutput testOutput = new TestOutput();
+ LogState logState = TCMDump.DumpMetadata.getLogState(storage, snapshotManager, null, testOutput.getOutput());
+ String stderr = testOutput.getStderr();
+
+ // Should detect gap at beginning (expected 1 but found 3)
+ assertThat(stderr).contains("Gap detected");
+ assertThat(stderr).contains("expected epoch 1 but found 3");
+
+ // All entries should still be present
+ assertThat(logState.entries).hasSize(3);
+ assertThat(logState.entries.get(0).epoch.getEpoch()).isEqualTo(3);
+ assertThat(logState.entries.get(1).epoch.getEpoch()).isEqualTo(4);
+ assertThat(logState.entries.get(2).epoch.getEpoch()).isEqualTo(5);
+ }
+
+ @Test
+ public void testTargetEpochFilter()
+ {
+ // Write entries 1-10
+ for (int i = 1; i <= 10; i++)
+ {
+ storage.append(entry(i));
+ }
+
+ // Get log state up to epoch 5
+ TestOutput testOutput = new TestOutput();
+ LogState logState = TCMDump.DumpMetadata.getLogState(storage, snapshotManager, 5L, testOutput.getOutput());
+ String stderr = testOutput.getStderr();
+
+ // No gaps
+ assertThat(stderr).doesNotContain("Gap detected");
+
+ // Should only have epochs up to 5
+ assertThat(logState.entries).hasSizeLessThanOrEqualTo(5);
+ for (Entry e : logState.entries)
+ {
+ assertThat(e.epoch.getEpoch()).isLessThanOrEqualTo(5);
+ }
+ }
+
+ /**
+ * Helper class to capture output from the gap detection logic.
+ */
+ private static class TestOutput
+ {
+ private final ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+ private final ByteArrayOutputStream errStream = new ByteArrayOutputStream();
+ private final Output output = new Output(new PrintStream(outStream), new PrintStream(errStream));
+
+ public Output getOutput()
+ {
+ return output;
+ }
+
+ public String getStdout()
+ {
+ return outStream.toString();
+ }
+
+ public String getStderr()
+ {
+ return errStream.toString();
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/tools/TCMDumpTest.java b/test/unit/org/apache/cassandra/tools/TCMDumpTest.java
new file mode 100644
index 000000000000..df00880ec529
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/TCMDumpTest.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import org.assertj.core.api.Assertions;
+import org.hamcrest.CoreMatchers;
+import org.junit.Test;
+
+import org.apache.cassandra.tools.ToolRunner.ToolResult;
+
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for TCMDump tool.
+ *
+ * Note: This tool requires some initialization (DatabaseDescriptor, Schema) even for help,
+ * similar to StandaloneJournalUtil and other TCM-related tools.
+ */
+public class TCMDumpTest extends OfflineToolUtils
+{
+ @Test
+ public void testMainHelpOption()
+ {
+ // Main command help shows subcommands
+ ToolResult tool = ToolRunner.invokeClass(TCMDump.class, "-h");
+ String output = tool.getStdout() + tool.getStderr();
+ assertThat("Help should show usage", output, CoreMatchers.containsStringIgnoringCase("Usage:"));
+ assertThat("Help should mention dump subcommand", output, CoreMatchers.containsStringIgnoringCase("dump"));
+ }
+
+ @Test
+ public void testDumpSubcommandHelpOption()
+ {
+ // Dump subcommand help shows all the options
+ ToolResult tool = ToolRunner.invokeClass(TCMDump.class, "dump", "-h");
+ String output = tool.getStdout() + tool.getStderr();
+
+ assertThat("Help should show usage", output, CoreMatchers.containsStringIgnoringCase("Usage:"));
+ // Check for key options
+ Assertions.assertThat(output).containsIgnoringCase("--data-dir");
+ Assertions.assertThat(output).containsIgnoringCase("--to-string");
+ Assertions.assertThat(output).containsIgnoringCase("--dump-log");
+ Assertions.assertThat(output).containsIgnoringCase("--dump-distributed-log");
+ }
+
+ @Test
+ public void testMaybeChangeDocs()
+ {
+ // If you added, modified options or help, please update docs if necessary
+ ToolResult tool = ToolRunner.invokeClass(TCMDump.class, "dump", "-h");
+ String output = tool.getStdout() + tool.getStderr();
+
+ // Verify key options are documented
+ Assertions.assertThat(output).containsIgnoringCase("--data-dir");
+ Assertions.assertThat(output).containsIgnoringCase("--sstables");
+ Assertions.assertThat(output).containsIgnoringCase("--partitioner");
+ Assertions.assertThat(output).containsIgnoringCase("--output");
+ Assertions.assertThat(output).containsIgnoringCase("--to-string");
+ Assertions.assertThat(output).containsIgnoringCase("--dump-log");
+ Assertions.assertThat(output).containsIgnoringCase("--dump-distributed-log");
+ Assertions.assertThat(output).containsIgnoringCase("--epoch");
+ Assertions.assertThat(output).containsIgnoringCase("--from-epoch");
+ Assertions.assertThat(output).containsIgnoringCase("--to-epoch");
+ Assertions.assertThat(output).containsIgnoringCase("--verbose");
+ Assertions.assertThat(output).containsIgnoringCase("--debug");
+ }
+
+ @Test
+ public void testWrongArgFailsAndPrintsHelp()
+ {
+ ToolResult tool = ToolRunner.invokeClass(TCMDump.class, "dump", "--invalid-option");
+ String output = tool.getStdout() + tool.getStderr();
+ assertThat("Should mention unknown option", output, CoreMatchers.containsStringIgnoringCase("Unknown"));
+ assertTrue("Expected non-zero exit code", tool.getExitCode() != 0);
+ }
+
+ @Test
+ public void testNonExistentDataDirectory()
+ {
+ // When running with a non-existent directory, should fail gracefully
+ ToolResult tool = ToolRunner.invokeClass(TCMDump.class, "dump",
+ "--data-dir", "/nonexistent/path/to/data",
+ "--to-string");
+ String output = tool.getStdout() + tool.getStderr();
+ // Tool should fail gracefully when directory doesn't exist or no SSTables found
+ assertTrue("Expected error or no metadata message",
+ tool.getExitCode() != 0 ||
+ output.toLowerCase().contains("no metadata") ||
+ output.toLowerCase().contains("not found") ||
+ output.toLowerCase().contains("does not exist") ||
+ output.toLowerCase().contains("error"));
+ }
+
+ @Test
+ public void testOutputModeFlags()
+ {
+ // Test that --to-string flag is recognized
+ ToolResult toStringFlag = ToolRunner.invokeClass(TCMDump.class, "dump", "--to-string", "-h");
+ String toStringOutput = toStringFlag.getStdout() + toStringFlag.getStderr();
+ assertThat("Should show help with --to-string", toStringOutput, CoreMatchers.containsStringIgnoringCase("Usage:"));
+
+ // Test that --dump-log flag is recognized
+ ToolResult dumpLogFlag = ToolRunner.invokeClass(TCMDump.class, "dump", "--dump-log", "-h");
+ String dumpLogOutput = dumpLogFlag.getStdout() + dumpLogFlag.getStderr();
+ assertThat("Should show help with --dump-log", dumpLogOutput, CoreMatchers.containsStringIgnoringCase("Usage:"));
+
+ // Test that --dump-distributed-log flag is recognized
+ ToolResult distLogFlag = ToolRunner.invokeClass(TCMDump.class, "dump", "--dump-distributed-log", "-h");
+ String distLogOutput = distLogFlag.getStdout() + distLogFlag.getStderr();
+ assertThat("Should show help with --dump-distributed-log", distLogOutput, CoreMatchers.containsStringIgnoringCase("Usage:"));
+
+ // Test that -o/--output flag is recognized
+ ToolResult outputFlag = ToolRunner.invokeClass(TCMDump.class, "dump", "-o", "/tmp/test.dump", "-h");
+ String outputOutput = outputFlag.getStdout() + outputFlag.getStderr();
+ assertThat("Should show help with -o", outputOutput, CoreMatchers.containsStringIgnoringCase("Usage:"));
+
+ ToolResult outputLongFlag = ToolRunner.invokeClass(TCMDump.class, "dump", "--output", "/tmp/test.dump", "-h");
+ String outputLongOutput = outputLongFlag.getStdout() + outputLongFlag.getStderr();
+ assertThat("Should show help with --output", outputLongOutput, CoreMatchers.containsStringIgnoringCase("Usage:"));
+ }
+
+ @Test
+ public void testEpochFilterFlags()
+ {
+ // Test that epoch filter flags are recognized
+ ToolResult epochTool = ToolRunner.invokeClass(TCMDump.class, "dump", "--epoch", "100", "-h");
+ String epochOutput = epochTool.getStdout() + epochTool.getStderr();
+ assertThat("--epoch flag should be recognized", epochOutput, CoreMatchers.containsStringIgnoringCase("Usage:"));
+
+ ToolResult fromTool = ToolRunner.invokeClass(TCMDump.class, "dump", "--from-epoch", "50", "-h");
+ String fromOutput = fromTool.getStdout() + fromTool.getStderr();
+ assertThat("--from-epoch flag should be recognized", fromOutput, CoreMatchers.containsStringIgnoringCase("Usage:"));
+
+ ToolResult toTool = ToolRunner.invokeClass(TCMDump.class, "dump", "--to-epoch", "150", "-h");
+ String toOutput = toTool.getStdout() + toTool.getStderr();
+ assertThat("--to-epoch flag should be recognized", toOutput, CoreMatchers.containsStringIgnoringCase("Usage:"));
+ }
+
+ @Test
+ public void testVerboseAndDebugFlags()
+ {
+ // Test verbose flags
+ ToolResult verboseShort = ToolRunner.invokeClass(TCMDump.class, "dump", "-v", "-h");
+ String verboseShortOutput = verboseShort.getStdout() + verboseShort.getStderr();
+ assertThat("-v flag should be recognized", verboseShortOutput, CoreMatchers.containsStringIgnoringCase("Usage:"));
+
+ ToolResult verboseLong = ToolRunner.invokeClass(TCMDump.class, "dump", "--verbose", "-h");
+ String verboseLongOutput = verboseLong.getStdout() + verboseLong.getStderr();
+ assertThat("--verbose flag should be recognized", verboseLongOutput, CoreMatchers.containsStringIgnoringCase("Usage:"));
+
+ // Test debug flag
+ ToolResult debug = ToolRunner.invokeClass(TCMDump.class, "dump", "--debug", "-h");
+ String debugOutput = debug.getStdout() + debug.getStderr();
+ assertThat("--debug flag should be recognized", debugOutput, CoreMatchers.containsStringIgnoringCase("Usage:"));
+ }
+
+ @Test
+ public void testPartitionerFlag()
+ {
+ // Test partitioner flags
+ ToolResult shortFlag = ToolRunner.invokeClass(TCMDump.class, "dump",
+ "-p", "org.apache.cassandra.dht.Murmur3Partitioner", "-h");
+ String shortOutput = shortFlag.getStdout() + shortFlag.getStderr();
+ assertThat("-p flag should be recognized", shortOutput, CoreMatchers.containsStringIgnoringCase("Usage:"));
+
+ ToolResult longFlag = ToolRunner.invokeClass(TCMDump.class, "dump",
+ "--partitioner", "org.apache.cassandra.dht.Murmur3Partitioner", "-h");
+ String longOutput = longFlag.getStdout() + longFlag.getStderr();
+ assertThat("--partitioner flag should be recognized", longOutput, CoreMatchers.containsStringIgnoringCase("Usage:"));
+ }
+}
diff --git a/tools/bin/tcmdump b/tools/bin/tcmdump
new file mode 100755
index 000000000000..caec66cf0314
--- /dev/null
+++ b/tools/bin/tcmdump
@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
+ # Locations (in order) to use when searching for an include file.
+ for include in "`dirname "$0"`/cassandra.in.sh" \
+ "$HOME/.cassandra.in.sh" \
+ /usr/share/cassandra/cassandra.in.sh \
+ /usr/local/share/cassandra/cassandra.in.sh \
+ /opt/cassandra/cassandra.in.sh; do
+ if [ -r "$include" ]; then
+ . "$include"
+ break
+ fi
+ done
+elif [ -r "$CASSANDRA_INCLUDE" ]; then
+ . "$CASSANDRA_INCLUDE"
+fi
+
+if [ -z "$CLASSPATH" ]; then
+ echo "You must set the CLASSPATH var" >&2
+ exit 1
+fi
+
+if [ "x$MAX_HEAP_SIZE" = "x" ]; then
+ MAX_HEAP_SIZE="256M"
+fi
+
+"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
+ -Dcassandra.storagedir="$cassandra_storagedir" \
+ -Dlogback.configurationFile=logback-tools.xml \
+ org.apache.cassandra.tools.TCMDump "$@"
+
+# vi:ai sw=4 ts=4 tw=0 et