improve: speed up pull-db and add restore-db command

Dockerfile

@@ -68,6 +68,7 @@ COPY --from=build /upflow/tsconfig.json /upflow/tsconfig.json
 COPY --from=build /upflow/start.sh /upflow/start.sh
 COPY --from=build /upflow/app /upflow/app
 COPY --from=build /upflow/batch /upflow/batch
+COPY --from=build /upflow/ops/remote /upflow/ops/remote
 COPY --from=build /upflow/server.mjs /upflow/server.mjs
 
 CMD [ "sh", "./start.sh" ]

ops/cli.ts

@@ -1,5 +1,6 @@
 import { cli, command } from 'cleye'
 import { pullDbCommand } from './commands/pull-db'
+import { restoreDbCommand } from './commands/restore-db'
 
 const pullDb = command(
   {
@@ -32,7 +33,31 @@ const pullDb = command(
   },
 )
 
-cli({
+const restoreDb = command(
+  {
+    name: 'restore-db',
+    flags: {
+      name: {
+        type: String,
+        description:
+          'Backup name to restore (e.g. backup_2026-03-15T14-18-50-365Z). Omit to list available backups.',
+      },
+    },
+    help: {
+      description: 'Restore database files from a previous backup.',
+    },
+  },
+  (argv) => {
+    const { help, ...flags } = argv.flags
+    restoreDbCommand(flags)
+  },
+)
+
+const argv = cli({
   name: 'ops',
-  commands: [pullDb],
+  commands: [pullDb, restoreDb],
 })
+
+if (!argv.command) {
+  argv.showHelp()
+}

ops/commands/pull-db.ts

@@ -1,13 +1,11 @@
 import Database from 'better-sqlite3'
 import consola from 'consola'
-import { execSync } from 'node:child_process'
+import { execFileSync } from 'node:child_process'
 import fs from 'node:fs'
 import path from 'node:path'
+import { BACKUP_PREFIX, DATA_DIR, isDbFile } from '../lib/data-dir'
 
-const DATA_DIR = 'data'
-const REMOTE_DATA_DIR = '/upflow/data'
 const VALID_APP_NAME = /^[a-z0-9-]+$/
-const VALID_DB_FILENAME = /^[\w.-]+\.db$/
 
 interface PullDbOptions {
   app: string
@@ -28,94 +26,83 @@ function backupExistingData() {
     return
   }
 
+  const dbFiles = fs.readdirSync(DATA_DIR).filter(isDbFile)
+  if (dbFiles.length === 0) {
+    consola.info('No database files to back up')
+    return
+  }
+
   const timestamp = new Date().toISOString().replace(/[:.]/g, '-')
-  const backupDir = path.join(DATA_DIR, `backup_${timestamp}`)
+  const backupDir = path.join(DATA_DIR, `${BACKUP_PREFIX}${timestamp}`)
   fs.mkdirSync(backupDir, { recursive: true })
 
-  const files = fs
-    .readdirSync(DATA_DIR)
-    .filter(
-      (f) =>
-        f.endsWith('.db') || f.endsWith('.db-wal') || f.endsWith('.db-shm'),
-    )
-  for (const file of files) {
-    const src = path.join(DATA_DIR, file)
-    const dest = path.join(backupDir, file)
-    fs.copyFileSync(src, dest)
-    consola.debug(`Backed up ${file}`)
-  }
-  consola.success(`Backed up ${files.length} files to ${backupDir}`)
-}
-
-function listRemoteTenantDbs(app: string): string[] {
-  consola.start('Listing remote tenant databases...')
-  try {
-    const output = execSync(
-      `fly ssh console -a ${app} -C "sh -c 'ls ${REMOTE_DATA_DIR}/tenant_*.db'"`,
-      { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] },
-    )
-    return output
-      .trim()
-      .split('\n')
-      .filter(Boolean)
-      .map((line) => path.basename(line.trim()))
-      .filter((name) => VALID_DB_FILENAME.test(name))
-  } catch (error) {
-    const stderr =
-      error instanceof Error && 'stderr' in error
-        ? String((error as { stderr: unknown }).stderr)
-        : ''
-    consola.warn(
-      `No tenant databases found or command failed${stderr ? `: ${stderr.trim()}` : ''}`,
-    )
-    return []
+  for (const file of dbFiles) {
+    fs.copyFileSync(path.join(DATA_DIR, file), path.join(backupDir, file))
   }
+  consola.success(`Backed up ${dbFiles.length} files to ${backupDir}`)
 }
 
 /**
- * リモート DB の WAL をメイン DB ファイルにフラッシュする。
- * WAL モードで動作中の DB は -wal ファイルに未コミットの変更を持つため、
- * .db ファイルだけ pull すると不整合（orphan index 等）が発生する。
+ * リモートの prepare-pull.sh を実行して tar.gz を作成し、
+ * 1回の SFTP で取得して展開する。
  */
-function checkpointRemoteDbs(app: string, dbFiles: string[]) {
-  consola.start('Checkpointing remote databases (flushing WAL)...')
-  let succeeded = 0
-  for (const f of dbFiles) {
-    const dbPath = `${REMOTE_DATA_DIR}/${f}`
-    try {
-      execSync(
-        `fly ssh console -a ${app} -C 'sqlite3 ${dbPath} "PRAGMA wal_checkpoint(TRUNCATE);"'`,
-        { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] },
-      )
-      succeeded++
-    } catch {
-      consola.warn(
-        `WAL checkpoint failed for ${f}, will pull WAL files instead`,
-      )
-    }
-  }
-  if (succeeded > 0) {
-    consola.success(`Checkpointed ${succeeded}/${dbFiles.length} database(s)`)
-  }
-}
+function pullAllDbs(app: string) {
+  const remoteTar = '/tmp/upflow-data.tar.gz'
+  const localTar = path.join(DATA_DIR, '_pull.tar.gz')
+
+  // Step 1: リモートで .backup + tar.gz 作成
+  consola.start('Remote: creating atomic backups and archive...')
+  const output = execFileSync(
+    'fly',
+    [
+      'ssh',
+      'console',
+      '-a',
+      app,
+      '-C',
+      'sh /upflow/ops/remote/prepare-pull.sh',
+    ],
+    { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] },
+  )
 
-function pullFile(app: string, remoteFile: string, localFile: string) {
-  // fly ssh sftp get refuses to overwrite existing files
-  if (fs.existsSync(localFile)) {
-    fs.unlinkSync(localFile)
+  // Parse file list from output between ---FILES--- and ---DONE---
+  const lines = output.split('\n').map((l) => l.trim())
+  const startIdx = lines.indexOf('---FILES---')
+  const endIdx = lines.indexOf('---DONE---')
+  const dbFiles =
+    startIdx >= 0 && endIdx > startIdx
+      ? lines.slice(startIdx + 1, endIdx).filter(Boolean)
+      : []
+
+  if (dbFiles.length === 0) {
+    consola.error('No database files found on remote')
+    process.exit(1)
   }
-  consola.start(`Pulling ${remoteFile}...`)
-  execSync(`fly ssh sftp get -a ${app} ${remoteFile} ${localFile}`, {
-    stdio: 'inherit',
-  })
-}
+  consola.success(
+    `Remote archive ready: ${dbFiles.length} database(s) — ${dbFiles.join(', ')}`,
+  )
 
-function tryPullFile(app: string, remoteFile: string, localFile: string) {
+  // Step 2: 1回の SFTP で tar.gz を取得
+  consola.start('Pulling archive...')
   try {
-    pullFile(app, remoteFile, localFile)
+    fs.unlinkSync(localTar)
   } catch {
-    // WAL/SHM files may not exist (e.g. after checkpoint), ignore
+    // file may not exist
   }
+  execFileSync('fly', ['ssh', 'sftp', 'get', '-a', app, remoteTar, localTar], {
+    stdio: 'inherit',
+  })
+
+  // Step 3: ローカルで展開
+  consola.start('Extracting archive...')
+  execFileSync('tar', ['xzf', path.resolve(localTar)], {
+    cwd: DATA_DIR,
+    stdio: 'inherit',
+  })
+  fs.unlinkSync(localTar)
+
+  consola.success(`Pulled ${dbFiles.length} database(s)`)
+  return dbFiles
 }
 
 function sanitizeExportSettings(dbPath: string) {
@@ -158,56 +145,21 @@ export function pullDbCommand(options: PullDbOptions) {
   // Ensure data directory exists
   fs.mkdirSync(DATA_DIR, { recursive: true })
 
-  // Step 2: List tenant databases
-  const tenantDbs = listRemoteTenantDbs(app)
-  const filesToPull = ['data.db', ...tenantDbs]
-
-  consola.info(
-    `Found ${filesToPull.length} database(s): ${filesToPull.join(', ')}`,
-  )
-
-  // Step 3: Checkpoint WAL on remote before pulling
-  checkpointRemoteDbs(app, filesToPull)
-
-  // Step 4: Pull each file (+ WAL/SHM as fallback)
-  let pulledCount = 0
-  for (const file of filesToPull) {
-    try {
-      pullFile(app, `${REMOTE_DATA_DIR}/${file}`, path.join(DATA_DIR, file))
-      // Also pull WAL and SHM files if they exist (safety net)
-      tryPullFile(
-        app,
-        `${REMOTE_DATA_DIR}/${file}-wal`,
-        path.join(DATA_DIR, `${file}-wal`),
-      )
-      tryPullFile(
-        app,
-        `${REMOTE_DATA_DIR}/${file}-shm`,
-        path.join(DATA_DIR, `${file}-shm`),
-      )
-      pulledCount++
-    } catch (error) {
-      consola.error(`Failed to pull ${file}:`, error)
-    }
-  }
-
-  consola.success(`Pulled ${pulledCount}/${filesToPull.length} database(s)`)
+  // Step 2: Remote backup + tar + pull + extract
+  const dbFiles = pullAllDbs(app)
 
-  // Step 5: Sanitize export settings in all pulled databases
+  // Step 3: Sanitize export settings
   if (!noSanitize) {
     consola.start('Sanitizing export settings...')
-    for (const file of filesToPull) {
-      const dbPath = path.join(DATA_DIR, file)
-      if (fs.existsSync(dbPath)) {
-        sanitizeExportSettings(dbPath)
-      }
+    for (const file of dbFiles) {
+      sanitizeExportSettings(path.join(DATA_DIR, file))
     }
     consola.success('Sanitization complete')
   }
 
   consola.box(
     [
-      `Pull complete: ${pulledCount} file(s)`,
+      `Pull complete: ${dbFiles.length} database(s)`,
       noSanitize ? 'Sanitization: skipped' : 'Sanitization: done',
       noBackup ? 'Backup: skipped' : 'Backup: done',
     ].join('\n'),

ops/commands/restore-db.ts

@@ -0,0 +1,83 @@
+import consola from 'consola'
+import fs from 'node:fs'
+import path from 'node:path'
+import { BACKUP_PREFIX, DATA_DIR, isDbFile } from '../lib/data-dir'
+
+interface RestoreDbOptions {
+  name?: string
+}
+
+function listBackups(): string[] {
+  if (!fs.existsSync(DATA_DIR)) return []
+  return fs
+    .readdirSync(DATA_DIR, { withFileTypes: true })
+    .filter((d) => d.isDirectory() && d.name.startsWith(BACKUP_PREFIX))
+    .map((d) => d.name)
+    .sort()
+    .reverse()
+}
+
+export async function restoreDbCommand(options: RestoreDbOptions) {
+  const backups = listBackups()
+  if (backups.length === 0) {
+    consola.error('No backups found in data/')
+    process.exit(1)
+  }
+
+  const target = options.name
+
+  if (!target) {
+    consola.info('Available backups (newest first):')
+    for (const b of backups) {
+      const files = fs
+        .readdirSync(path.join(DATA_DIR, b))
+        .filter((f) => f.endsWith('.db'))
+      consola.log(`  ${b}  (${files.length} database(s))`)
+    }
+    consola.info('\nUsage: pnpm ops restore-db --name <backup_name>')
+    return
+  }
+
+  const backupDir = path.join(DATA_DIR, target)
+  const stat = fs.statSync(backupDir, { throwIfNoEntry: false })
+  if (!stat?.isDirectory()) {
+    consola.error(`Backup not found: ${target}`)
+    consola.info('Run without --name to list available backups')
+    process.exit(1)
+  }
+
+  const filesToRestore = fs.readdirSync(backupDir).filter(isDbFile)
+
+  if (filesToRestore.length === 0) {
+    consola.error(`No database files in backup: ${target}`)
+    process.exit(1)
+  }
+
+  const dbCount = filesToRestore.filter((f) => f.endsWith('.db')).length
+  consola.warn(
+    `This will replace all current database files with ${dbCount} database(s) from ${target}.`,
+  )
+  consola.warn(
+    'Ensure no batch jobs or dev server are accessing the databases.',
+  )
+  const confirmed = await consola.prompt('Continue with restore?', {
+    type: 'confirm',
+  })
+  if (!confirmed) {
+    consola.info('Restore cancelled')
+    return
+  }
+
+  // Remove current db files
+  const currentDbFiles = fs.readdirSync(DATA_DIR).filter(isDbFile)
+  for (const f of currentDbFiles) {
+    fs.unlinkSync(path.join(DATA_DIR, f))
+  }
+
+  // Copy backup files to data/
+  for (const f of filesToRestore) {
+    fs.copyFileSync(path.join(backupDir, f), path.join(DATA_DIR, f))
+  }
+
+  consola.success(`Restored ${dbCount} database(s) from ${target}`)
+}

ops/lib/data-dir.ts

@@ -0,0 +1,10 @@
+export const DATA_DIR = 'data'
+export const BACKUP_PREFIX = 'backup_'
+
+export function isDbFile(filename: string): boolean {
+  return (
+    filename.endsWith('.db') ||
+    filename.endsWith('.db-wal') ||
+    filename.endsWith('.db-shm')
+  )
+}