From 85b1104190cab4ff144a5d9fecebbc446d8e20ab Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Thu, 8 Jan 2026 04:50:03 +0000 Subject: [PATCH] feat(pgpm): add slice command for modularizing migration plans - Add core slicing logic in pgpm/core/src/slice/ with path-based grouping - Implement folder-based strategy to extract package names from change paths - Add dependency graph building, validation, and topological sorting - Generate cross-package dependencies using package:change syntax - Add slice command to pgpm CLI with dry-run and configuration options - Include comprehensive test suite with 18 passing tests - Add detailed design specification document --- pgpm/MIGRATION_SLICING_SPEC.md | 1368 +++++++++++++++++++++++ pgpm/cli/src/commands.ts | 14 +- pgpm/cli/src/commands/slice.ts | 220 ++++ pgpm/cli/src/index.ts | 1 + pgpm/core/__tests__/slice/slice.test.ts | 302 +++++ pgpm/core/src/index.ts | 1 + pgpm/core/src/slice/index.ts | 3 + pgpm/core/src/slice/output.ts | 174 +++ pgpm/core/src/slice/slice.ts | 637 +++++++++++ pgpm/core/src/slice/types.ts | 145 +++ 10 files changed, 2859 insertions(+), 6 deletions(-) create mode 100644 pgpm/MIGRATION_SLICING_SPEC.md create mode 100644 pgpm/cli/src/commands/slice.ts create mode 100644 pgpm/core/__tests__/slice/slice.test.ts create mode 100644 pgpm/core/src/slice/index.ts create mode 100644 pgpm/core/src/slice/output.ts create mode 100644 pgpm/core/src/slice/slice.ts create mode 100644 pgpm/core/src/slice/types.ts diff --git a/pgpm/MIGRATION_SLICING_SPEC.md b/pgpm/MIGRATION_SLICING_SPEC.md new file mode 100644 index 000000000..83b6b62da --- /dev/null +++ b/pgpm/MIGRATION_SLICING_SPEC.md @@ -0,0 +1,1368 @@ +# PGPM Migration Slicing Specification + +## Overview + +This document specifies an algorithm and approach for "slicing" a large PGPM plan file into multiple folder-based subpackages. The goal is to take a monolithic migration set (potentially thousands of changes) and partition it into modular, maintainable packages while preserving correctness and deployment order. + +## Problem Statement + +### Current State +- A single PGPM module with one `pgpm.plan` file containing N changes +- Changes have dependencies forming a DAG (Directed Acyclic Graph) +- All migrations run in linear order defined by the plan +- Large projects become unwieldy with thousands of files in one package + +### Desired State +- Multiple PGPM packages (e.g., `core/`, `auth/`, `rls/`, `graphql/`) +- Each package has its own `pgpm.plan` and `.control` file +- Cross-package dependencies expressed via `package:change` or `package:@tag` syntax +- Workspace-level manifest describing package relationships +- Deployment order preserved across the entire system + +## Data Structures + +### Input: Single Plan File + +``` +%syntax-version=1.0.0 +%project=myapp +%uri=myapp + +schemas/public 2024-01-01T00:00:00Z author # create public schema +extensions/uuid-ossp [schemas/public] 2024-01-01T00:00:01Z author +schemas/auth/tables/users [extensions/uuid-ossp] 2024-01-01T00:00:02Z author +schemas/auth/tables/sessions [schemas/auth/tables/users] 2024-01-01T00:00:03Z author +schemas/auth/functions/authenticate [schemas/auth/tables/users schemas/auth/tables/sessions] 2024-01-01T00:00:04Z author +schemas/graphql/tables/queries [schemas/auth/tables/users] 2024-01-01T00:00:05Z author +schemas/rls/policies/user_access [schemas/auth/tables/users schemas/graphql/tables/queries] 2024-01-01T00:00:06Z author +@v1.0.0 2024-01-01T00:00:07Z author # initial release +``` + +### Output: Multiple Packages + +``` +workspace/ +├── pgpm.json # Workspace manifest +├── packages/ +│ ├── core/ +│ │ ├── pgpm.plan +│ │ ├── core.control +│ │ ├── deploy/ +│ │ │ ├── schemas/public.sql +│ │ │ └── extensions/uuid-ossp.sql +│ │ ├── revert/ +│ │ └── verify/ +│ ├── auth/ +│ │ ├── pgpm.plan # deps: [core:@v1.0.0] +│ │ ├── auth.control # requires: core +│ │ ├── deploy/ +│ │ │ └── schemas/auth/... +│ │ ├── revert/ +│ │ └── verify/ +│ ├── graphql/ +│ │ ├── pgpm.plan # deps: [auth:@v1.0.0] +│ │ └── ... +│ └── rls/ +│ ├── pgpm.plan # deps: [auth:@v1.0.0, graphql:@v1.0.0] +│ └── ... +``` + +### Core Types (TypeScript) + +```typescript +interface SlicingConfig { + /** Source plan file path */ + sourcePlan: string; + + /** Output directory for packages */ + outputDir: string; + + /** Grouping strategy */ + strategy: GroupingStrategy; + + /** Optional explicit mapping overrides */ + explicitMapping?: Record; + + /** Package for changes that don't match any group */ + defaultPackage?: string; + + /** Whether to use tags for cross-package deps */ + useTagsForCrossPackageDeps?: boolean; + + /** Minimum changes per package (merge smaller groups) */ + minChangesPerPackage?: number; +} + +type GroupingStrategy = + | { type: 'folder'; depth: number; prefixToStrip?: string } + | { type: 'schema'; schemaExtractor: (changeName: string) => string } + | { type: 'explicit'; mapping: Record } + | { type: 'community'; maxPackages?: number } + | { type: 'custom'; grouper: (change: Change) => string }; + +interface SlicingResult { + /** Generated packages */ + packages: PackageOutput[]; + + /** Workspace manifest */ + workspace: WorkspaceManifest; + + /** Warnings/issues encountered */ + warnings: SlicingWarning[]; + + /** Statistics */ + stats: SlicingStats; +} + +interface PackageOutput { + name: string; + planContent: string; + controlContent: string; + changes: Change[]; + internalDeps: string[]; + externalDeps: string[]; +} + +interface WorkspaceManifest { + packages: string[]; + deployOrder: string[]; + dependencies: Record; +} + +interface SlicingWarning { + type: 'heavy_cross_deps' | 'cycle_detected' | 'orphan_change' | 'merge_required'; + message: string; + affectedChanges?: string[]; + suggestedAction?: string; +} + +interface SlicingStats { + totalChanges: number; + packagesCreated: number; + internalEdges: number; + crossPackageEdges: number; + crossPackageRatio: number; +} +``` + +## Algorithm + +### Phase 1: Parse and Build Dependency Graph + +```typescript +function buildDependencyGraph(planPath: string): DependencyGraph { + const planResult = parsePlanFile(planPath); + if (!planResult.data) { + throw new Error(`Failed to parse plan: ${planResult.errors}`); + } + + const { changes, tags } = planResult.data; + const graph: DependencyGraph = { + nodes: new Map(), + edges: new Map(), + reverseEdges: new Map(), + tags: new Map() + }; + + // Add all changes as nodes + for (const change of changes) { + graph.nodes.set(change.name, change); + graph.edges.set(change.name, new Set(change.dependencies)); + + // Build reverse edges for dependency analysis + for (const dep of change.dependencies) { + if (!graph.reverseEdges.has(dep)) { + graph.reverseEdges.set(dep, new Set()); + } + graph.reverseEdges.get(dep)!.add(change.name); + } + } + + // Index tags by change + for (const tag of tags) { + if (!graph.tags.has(tag.change)) { + graph.tags.set(tag.change, []); + } + graph.tags.get(tag.change)!.push(tag); + } + + // Validate DAG (detect cycles) + validateDAG(graph); + + return graph; +} + +function validateDAG(graph: DependencyGraph): void { + const visited = new Set(); + const visiting = new Set(); + + function dfs(node: string, path: string[]): void { + if (visiting.has(node)) { + throw new Error(`Cycle detected: ${[...path, node].join(' -> ')}`); + } + if (visited.has(node)) return; + + visiting.add(node); + const deps = graph.edges.get(node) || new Set(); + for (const dep of deps) { + dfs(dep, [...path, node]); + } + visiting.delete(node); + visited.add(node); + } + + for (const node of graph.nodes.keys()) { + dfs(node, []); + } +} +``` + +### Phase 2: Assign Changes to Packages + +```typescript +function assignChangesToPackages( + graph: DependencyGraph, + config: SlicingConfig +): Map> { + const assignments = new Map>(); + + // Apply grouping strategy + for (const [changeName, change] of graph.nodes) { + let packageName: string; + + // Check explicit mapping first + if (config.explicitMapping?.[changeName]) { + packageName = config.explicitMapping[changeName]; + } else { + packageName = applyGroupingStrategy(changeName, change, config.strategy); + } + + // Fallback to default package + if (!packageName) { + packageName = config.defaultPackage || 'core'; + } + + if (!assignments.has(packageName)) { + assignments.set(packageName, new Set()); + } + assignments.get(packageName)!.add(changeName); + } + + // Merge small packages if configured + if (config.minChangesPerPackage) { + mergeSmallPackages(assignments, config.minChangesPerPackage); + } + + return assignments; +} + +function applyGroupingStrategy( + changeName: string, + change: Change, + strategy: GroupingStrategy +): string { + switch (strategy.type) { + case 'folder': { + // Extract package from folder path + // e.g., "schemas/auth/tables/users" with depth=2 -> "auth" + const parts = changeName.split('/'); + const prefix = strategy.prefixToStrip || 'schemas'; + + // Strip prefix if present + let startIdx = 0; + if (parts[0] === prefix) { + startIdx = 1; + } + + // Handle special folders + if (parts[0] === 'extensions' || parts[0] === 'migrate') { + return 'core'; + } + + // Get package name at specified depth + if (parts.length > startIdx) { + return parts[startIdx]; + } + return 'core'; + } + + case 'schema': { + return strategy.schemaExtractor(changeName); + } + + case 'explicit': { + return strategy.mapping[changeName] || 'core'; + } + + case 'community': { + // Community detection would be applied separately + // This is a placeholder for the algorithm + return 'core'; + } + + case 'custom': { + return strategy.grouper(change); + } + } +} +``` + +### Phase 3: Validate Partition + +```typescript +interface ValidationResult { + valid: boolean; + errors: string[]; + warnings: SlicingWarning[]; + packageDependencies: Map>; +} + +function validatePartition( + graph: DependencyGraph, + assignments: Map> +): ValidationResult { + const errors: string[] = []; + const warnings: SlicingWarning[] = []; + const packageDeps = new Map>(); + + // Build reverse lookup: change -> package + const changeToPackage = new Map(); + for (const [pkg, changes] of assignments) { + for (const change of changes) { + changeToPackage.set(change, pkg); + } + } + + // Check each change's dependencies + for (const [changeName, deps] of graph.edges) { + const myPackage = changeToPackage.get(changeName)!; + + for (const dep of deps) { + const depPackage = changeToPackage.get(dep); + + if (!depPackage) { + // External dependency (from another workspace module) + continue; + } + + if (depPackage !== myPackage) { + // Cross-package dependency + if (!packageDeps.has(myPackage)) { + packageDeps.set(myPackage, new Set()); + } + packageDeps.get(myPackage)!.add(depPackage); + } + } + } + + // Check for cycles in package dependency graph + const pkgCycle = detectPackageCycle(packageDeps); + if (pkgCycle) { + errors.push(`Package cycle detected: ${pkgCycle.join(' -> ')}`); + } + + // Warn about heavy cross-package dependencies + for (const [pkg, changes] of assignments) { + let crossDeps = 0; + let totalDeps = 0; + + for (const change of changes) { + const deps = graph.edges.get(change) || new Set(); + for (const dep of deps) { + totalDeps++; + const depPkg = changeToPackage.get(dep); + if (depPkg && depPkg !== pkg) { + crossDeps++; + } + } + } + + const ratio = totalDeps > 0 ? crossDeps / totalDeps : 0; + if (ratio > 0.5) { + warnings.push({ + type: 'heavy_cross_deps', + message: `Package "${pkg}" has ${Math.round(ratio * 100)}% cross-package dependencies`, + suggestedAction: 'Consider merging with dependent packages or reorganizing' + }); + } + } + + return { + valid: errors.length === 0, + errors, + warnings, + packageDependencies: packageDeps + }; +} + +function detectPackageCycle(deps: Map>): string[] | null { + const visited = new Set(); + const visiting = new Set(); + + function dfs(pkg: string, path: string[]): string[] | null { + if (visiting.has(pkg)) { + return [...path, pkg]; + } + if (visited.has(pkg)) return null; + + visiting.add(pkg); + const pkgDeps = deps.get(pkg) || new Set(); + for (const dep of pkgDeps) { + const cycle = dfs(dep, [...path, pkg]); + if (cycle) return cycle; + } + visiting.delete(pkg); + visited.add(pkg); + return null; + } + + for (const pkg of deps.keys()) { + const cycle = dfs(pkg, []); + if (cycle) return cycle; + } + return null; +} +``` + +### Phase 4: Resolve Package Cycles (if needed) + +```typescript +function resolvePackageCycles( + graph: DependencyGraph, + assignments: Map>, + validation: ValidationResult +): Map> { + if (validation.valid) { + return assignments; + } + + // Strategy: Merge packages that form cycles + const newAssignments = new Map(assignments); + + // Find strongly connected components in package graph + const sccs = findSCCs(validation.packageDependencies); + + for (const scc of sccs) { + if (scc.length > 1) { + // Merge all packages in SCC into one + const mergedName = scc.sort().join('_'); + const mergedChanges = new Set(); + + for (const pkg of scc) { + const changes = newAssignments.get(pkg); + if (changes) { + for (const change of changes) { + mergedChanges.add(change); + } + newAssignments.delete(pkg); + } + } + + newAssignments.set(mergedName, mergedChanges); + } + } + + return newAssignments; +} +``` + +### Phase 5: Generate Package Order + +```typescript +function computeDeployOrder( + packageDeps: Map> +): string[] { + const order: string[] = []; + const visited = new Set(); + const allPackages = new Set(); + + // Collect all packages + for (const pkg of packageDeps.keys()) { + allPackages.add(pkg); + } + for (const deps of packageDeps.values()) { + for (const dep of deps) { + allPackages.add(dep); + } + } + + function visit(pkg: string): void { + if (visited.has(pkg)) return; + visited.add(pkg); + + const deps = packageDeps.get(pkg) || new Set(); + for (const dep of deps) { + visit(dep); + } + + order.push(pkg); + } + + // Visit all packages (sorted for determinism) + for (const pkg of [...allPackages].sort()) { + visit(pkg); + } + + return order; +} +``` + +### Phase 6: Generate Output + +```typescript +function generatePackageOutput( + graph: DependencyGraph, + assignments: Map>, + packageDeps: Map>, + config: SlicingConfig +): SlicingResult { + const packages: PackageOutput[] = []; + const warnings: SlicingWarning[] = []; + + // Build reverse lookup + const changeToPackage = new Map(); + for (const [pkg, changes] of assignments) { + for (const change of changes) { + changeToPackage.set(change, pkg); + } + } + + // Compute deploy order + const deployOrder = computeDeployOrder(packageDeps); + + // Generate each package + for (const pkgName of deployOrder) { + const changes = assignments.get(pkgName); + if (!changes) continue; + + const pkgOutput = generateSinglePackage( + pkgName, + changes, + graph, + changeToPackage, + packageDeps.get(pkgName) || new Set(), + config + ); + + packages.push(pkgOutput); + } + + // Calculate stats + let internalEdges = 0; + let crossPackageEdges = 0; + + for (const [change, deps] of graph.edges) { + const myPkg = changeToPackage.get(change); + for (const dep of deps) { + const depPkg = changeToPackage.get(dep); + if (depPkg === myPkg) { + internalEdges++; + } else if (depPkg) { + crossPackageEdges++; + } + } + } + + return { + packages, + workspace: { + packages: deployOrder, + deployOrder, + dependencies: Object.fromEntries( + [...packageDeps.entries()].map(([k, v]) => [k, [...v]]) + ) + }, + warnings, + stats: { + totalChanges: graph.nodes.size, + packagesCreated: packages.length, + internalEdges, + crossPackageEdges, + crossPackageRatio: crossPackageEdges / (internalEdges + crossPackageEdges) + } + }; +} + +function generateSinglePackage( + pkgName: string, + changes: Set, + graph: DependencyGraph, + changeToPackage: Map, + pkgDeps: Set, + config: SlicingConfig +): PackageOutput { + // Sort changes in topological order within package + const sortedChanges = topologicalSortWithinPackage(changes, graph); + + // Build plan entries + const planEntries: Change[] = []; + const externalDeps: string[] = []; + + for (const changeName of sortedChanges) { + const originalChange = graph.nodes.get(changeName)!; + const deps = graph.edges.get(changeName) || new Set(); + + const newDeps: string[] = []; + + for (const dep of deps) { + const depPkg = changeToPackage.get(dep); + + if (!depPkg) { + // External dependency (from installed module) + newDeps.push(dep); + if (!externalDeps.includes(dep)) { + externalDeps.push(dep); + } + } else if (depPkg === pkgName) { + // Internal dependency - keep as-is + newDeps.push(dep); + } else { + // Cross-package dependency + if (config.useTagsForCrossPackageDeps) { + // Find latest tag in the dependency package + const depPkgChanges = [...(graph.nodes.keys())] + .filter(c => changeToPackage.get(c) === depPkg); + const lastChange = depPkgChanges[depPkgChanges.length - 1]; + const tags = graph.tags.get(lastChange); + + if (tags && tags.length > 0) { + const latestTag = tags[tags.length - 1]; + newDeps.push(`${depPkg}:@${latestTag.name}`); + } else { + newDeps.push(`${depPkg}:${dep}`); + } + } else { + newDeps.push(`${depPkg}:${dep}`); + } + } + } + + planEntries.push({ + name: changeName, + dependencies: newDeps, + timestamp: originalChange.timestamp, + planner: originalChange.planner, + email: originalChange.email, + comment: originalChange.comment + }); + } + + // Generate plan content + const planContent = generatePlanContent(pkgName, planEntries); + + // Generate control file content + const controlContent = generateControlContent(pkgName, pkgDeps); + + return { + name: pkgName, + planContent, + controlContent, + changes: planEntries, + internalDeps: sortedChanges.filter(c => changes.has(c)), + externalDeps + }; +} + +function topologicalSortWithinPackage( + changes: Set, + graph: DependencyGraph +): string[] { + const result: string[] = []; + const visited = new Set(); + + function visit(change: string): void { + if (visited.has(change)) return; + if (!changes.has(change)) return; // Skip changes not in this package + + visited.add(change); + + const deps = graph.edges.get(change) || new Set(); + for (const dep of deps) { + if (changes.has(dep)) { + visit(dep); + } + } + + result.push(change); + } + + // Visit in alphabetical order for determinism + for (const change of [...changes].sort()) { + visit(change); + } + + return result; +} + +function generatePlanContent(pkgName: string, entries: Change[]): string { + let content = `%syntax-version=1.0.0\n`; + content += `%project=${pkgName}\n`; + content += `%uri=${pkgName}\n\n`; + + for (const entry of entries) { + let line = entry.name; + + if (entry.dependencies.length > 0) { + line += ` [${entry.dependencies.join(' ')}]`; + } + + if (entry.timestamp) { + line += ` ${entry.timestamp}`; + if (entry.planner) { + line += ` ${entry.planner}`; + if (entry.email) { + line += ` <${entry.email}>`; + } + } + } + + if (entry.comment) { + line += ` # ${entry.comment}`; + } + + content += line + '\n'; + } + + return content; +} + +function generateControlContent(pkgName: string, deps: Set): string { + let content = `# ${pkgName} extension\n`; + content += `comment = '${pkgName} module'\n`; + content += `default_version = '0.0.1'\n`; + content += `relocatable = false\n`; + + if (deps.size > 0) { + content += `requires = '${[...deps].sort().join(', ')}'\n`; + } + + return content; +} +``` + +## Correctness Conditions + +### 1. Completeness +Every change from the original plan must appear in exactly one output package. + +```typescript +function verifyCompleteness( + original: Set, + packages: PackageOutput[] +): boolean { + const allOutputChanges = new Set(); + + for (const pkg of packages) { + for (const change of pkg.changes) { + if (allOutputChanges.has(change.name)) { + throw new Error(`Duplicate change: ${change.name}`); + } + allOutputChanges.add(change.name); + } + } + + for (const change of original) { + if (!allOutputChanges.has(change)) { + throw new Error(`Missing change: ${change}`); + } + } + + return true; +} +``` + +### 2. Dependency Satisfaction +For each change C with dependency D: +- D is in the same package AND appears before C in the plan, OR +- D is in another package that is declared as a dependency + +```typescript +function verifyDependencies( + packages: PackageOutput[], + workspace: WorkspaceManifest +): boolean { + const changeLocation = new Map(); + const changeOrder = new Map(); + + // Build location and order maps + for (const pkg of packages) { + let order = 0; + for (const change of pkg.changes) { + changeLocation.set(change.name, pkg.name); + changeOrder.set(`${pkg.name}:${change.name}`, order++); + } + } + + // Verify each dependency + for (const pkg of packages) { + for (const change of pkg.changes) { + for (const dep of change.dependencies) { + // Parse dependency + const [depPkg, depChange] = dep.includes(':') + ? dep.split(':') + : [pkg.name, dep]; + + // Skip tag references (they're validated separately) + if (depChange.startsWith('@')) continue; + + if (depPkg === pkg.name) { + // Internal dependency - must appear before + const myOrder = changeOrder.get(`${pkg.name}:${change.name}`)!; + const depOrder = changeOrder.get(`${pkg.name}:${depChange}`); + + if (depOrder === undefined || depOrder >= myOrder) { + throw new Error( + `Invalid order: ${change.name} depends on ${depChange} but it appears later` + ); + } + } else { + // Cross-package dependency - package must be declared + const pkgDeps = workspace.dependencies[pkg.name] || []; + if (!pkgDeps.includes(depPkg)) { + throw new Error( + `Undeclared package dependency: ${pkg.name} -> ${depPkg}` + ); + } + } + } + } + } + + return true; +} +``` + +### 3. Package Acyclicity +The package dependency graph must be a DAG. + +### 4. Order Preservation +Within each package, the topological order of changes must be maintained. + +## Edge Cases and Handling + +### 1. Cycles Across Folders (Mutual Dependencies) + +**Problem**: Changes in folder A depend on changes in folder B, and vice versa. + +**Detection**: +```typescript +function detectCrossGroupCycles( + graph: DependencyGraph, + assignments: Map> +): string[][] { + // Build package-level graph and find SCCs + const pkgGraph = buildPackageGraph(graph, assignments); + return findSCCs(pkgGraph); +} +``` + +**Resolution**: +- Merge the cyclically-dependent packages into one +- Warn the user about the merge +- Suggest reorganization + +### 2. Heavy Cross-Package Dependencies + +**Problem**: A package has more than 50% of its dependencies pointing to other packages. + +**Detection**: Calculate cross-package dependency ratio during validation. + +**Resolution**: +- Warn the user +- Suggest merging with the most-depended-upon package +- Provide statistics to help decision-making + +### 3. Shared Objects (Types/Functions Used Everywhere) + +**Problem**: Some objects (like utility functions, common types) are used by many packages. + +**Detection**: +```typescript +function findHighFanoutChanges( + graph: DependencyGraph, + threshold: number = 5 +): string[] { + const fanout: string[] = []; + + for (const [change, dependents] of graph.reverseEdges) { + if (dependents.size >= threshold) { + fanout.push(change); + } + } + + return fanout; +} +``` + +**Resolution**: +- Place high-fanout changes in a "core" package +- Core package has no dependencies on other packages +- All other packages depend on core + +### 4. Boundary Objects (Extensions, Schemas) + +**Problem**: PostgreSQL extensions and schema definitions are foundational. + +**Detection**: Check for `extensions/` prefix or schema creation patterns. + +**Resolution**: +- Always place in "core" package +- Process these first in the grouping phase + +### 5. Renames/Reorganization Without Breaking History + +**Problem**: Moving changes between packages shouldn't break existing deployments. + +**Resolution**: +- Track original change names in metadata +- Generate migration scripts for existing deployments +- Use tags to mark stable points before reorganization + +### 6. When to Refuse Splitting + +**Conditions**: +- More than 70% of edges are cross-package +- Package cycle cannot be resolved without merging all packages +- Resulting packages would have fewer than N changes each + +**Action**: +- Return error with explanation +- Suggest alternative strategies +- Offer to proceed with warnings + +## Heuristics for Grouping + +### 1. Folder-Based (Primary) + +```typescript +function folderBasedGrouping(changeName: string, depth: number = 1): string { + const parts = changeName.split('/'); + + // Handle special prefixes + if (parts[0] === 'schemas' && parts.length > 1) { + return parts[1]; // e.g., "schemas/auth/..." -> "auth" + } + + if (parts[0] === 'extensions' || parts[0] === 'migrate') { + return 'core'; + } + + // Default: use first segment + return parts[0] || 'core'; +} +``` + +### 2. Schema-Based + +```typescript +function schemaBasedGrouping(changeName: string, sqlContent: string): string { + // Extract schema from SQL content + const schemaMatch = sqlContent.match(/CREATE\s+(?:TABLE|FUNCTION|TYPE)\s+(\w+)\./i); + if (schemaMatch) { + return schemaMatch[1]; + } + + // Fallback to folder-based + return folderBasedGrouping(changeName); +} +``` + +### 3. Dependency Community Detection + +```typescript +function communityBasedGrouping( + graph: DependencyGraph, + maxCommunities: number = 10 +): Map { + // Louvain algorithm for community detection + // Returns mapping of change -> community ID + + // Initialize: each node in its own community + const communities = new Map(); + let communityId = 0; + for (const node of graph.nodes.keys()) { + communities.set(node, communityId++); + } + + // Iteratively optimize modularity + let improved = true; + while (improved) { + improved = false; + + for (const node of graph.nodes.keys()) { + const currentCommunity = communities.get(node)!; + let bestCommunity = currentCommunity; + let bestGain = 0; + + // Try moving to neighbor communities + const neighbors = new Set(); + const deps = graph.edges.get(node) || new Set(); + const reverseDeps = graph.reverseEdges.get(node) || new Set(); + + for (const dep of [...deps, ...reverseDeps]) { + neighbors.add(communities.get(dep)!); + } + + for (const targetCommunity of neighbors) { + const gain = calculateModularityGain( + graph, communities, node, currentCommunity, targetCommunity + ); + + if (gain > bestGain) { + bestGain = gain; + bestCommunity = targetCommunity; + } + } + + if (bestCommunity !== currentCommunity) { + communities.set(node, bestCommunity); + improved = true; + } + } + } + + // Convert to package names + const result = new Map(); + const communityNames = new Map(); + + for (const [node, comm] of communities) { + if (!communityNames.has(comm)) { + communityNames.set(comm, `pkg_${communityNames.size}`); + } + result.set(node, communityNames.get(comm)!); + } + + return result; +} +``` + +### 4. Core-First Layering + +```typescript +function coreFirstLayering(graph: DependencyGraph): Map { + const result = new Map(); + const layers: string[][] = []; + const assigned = new Set(); + + // Layer 0: nodes with no dependencies (roots) + const layer0: string[] = []; + for (const [node, deps] of graph.edges) { + if (deps.size === 0) { + layer0.push(node); + assigned.add(node); + } + } + layers.push(layer0); + + // Build subsequent layers + while (assigned.size < graph.nodes.size) { + const nextLayer: string[] = []; + + for (const [node, deps] of graph.edges) { + if (assigned.has(node)) continue; + + // Check if all deps are assigned + let allDepsAssigned = true; + for (const dep of deps) { + if (!assigned.has(dep)) { + allDepsAssigned = false; + break; + } + } + + if (allDepsAssigned) { + nextLayer.push(node); + } + } + + for (const node of nextLayer) { + assigned.add(node); + } + + if (nextLayer.length > 0) { + layers.push(nextLayer); + } + } + + // Assign packages based on layers + // Layer 0-1: core + // Layer 2+: based on folder structure + for (let i = 0; i < layers.length; i++) { + for (const node of layers[i]) { + if (i <= 1) { + result.set(node, 'core'); + } else { + result.set(node, folderBasedGrouping(node)); + } + } + } + + return result; +} +``` + +## Complexity Analysis + +| Operation | Time Complexity | Space Complexity | +|-----------|-----------------|------------------| +| Parse plan file | O(N) | O(N) | +| Build dependency graph | O(N + E) | O(N + E) | +| Validate DAG | O(N + E) | O(N) | +| Assign to packages | O(N) | O(N) | +| Validate partition | O(N + E) | O(N) | +| Detect package cycles | O(P + PE) | O(P) | +| Compute deploy order | O(P + PE) | O(P) | +| Generate output | O(N + E) | O(N + E) | +| Community detection | O(N log N) | O(N) | + +Where: +- N = number of changes +- E = number of dependency edges +- P = number of packages +- PE = number of package-level edges + +**Total**: O(N log N + E) for community detection, O(N + E) otherwise. + +## Determinism and Stability + +To ensure consistent outputs across runs: + +1. **Sort changes alphabetically** within each package before processing +2. **Sort packages alphabetically** when generating workspace manifest +3. **Use stable sorting** for topological sort (prefer alphabetical order for ties) +4. **Deterministic timestamps**: Use original timestamps from source plan + +```typescript +function stableTopologicalSort( + changes: Set, + graph: DependencyGraph +): string[] { + const result: string[] = []; + const inDegree = new Map(); + const queue: string[] = []; + + // Initialize in-degrees + for (const change of changes) { + inDegree.set(change, 0); + } + + for (const change of changes) { + const deps = graph.edges.get(change) || new Set(); + for (const dep of deps) { + if (changes.has(dep)) { + inDegree.set(change, inDegree.get(change)! + 1); + } + } + } + + // Find initial nodes with in-degree 0 + for (const [change, degree] of inDegree) { + if (degree === 0) { + queue.push(change); + } + } + + // Sort queue alphabetically for determinism + queue.sort(); + + while (queue.length > 0) { + const change = queue.shift()!; + result.push(change); + + // Find dependents within this package + const dependents = graph.reverseEdges.get(change) || new Set(); + const newZeros: string[] = []; + + for (const dependent of dependents) { + if (!changes.has(dependent)) continue; + + const newDegree = inDegree.get(dependent)! - 1; + inDegree.set(dependent, newDegree); + + if (newDegree === 0) { + newZeros.push(dependent); + } + } + + // Sort and add to queue for determinism + newZeros.sort(); + queue.push(...newZeros); + } + + return result; +} +``` + +## Illustrative Example + +### Input: 10 Changes + +``` +%project=myapp + +schemas/public 2024-01-01T00:00:00Z +extensions/uuid [schemas/public] 2024-01-01T00:00:01Z +schemas/auth/types/user_role [schemas/public] 2024-01-01T00:00:02Z +schemas/auth/tables/users [extensions/uuid schemas/auth/types/user_role] 2024-01-01T00:00:03Z +schemas/auth/tables/sessions [schemas/auth/tables/users] 2024-01-01T00:00:04Z +schemas/auth/functions/login [schemas/auth/tables/users schemas/auth/tables/sessions] 2024-01-01T00:00:05Z +schemas/api/tables/requests [schemas/auth/tables/users] 2024-01-01T00:00:06Z +schemas/api/functions/handle [schemas/api/tables/requests] 2024-01-01T00:00:07Z +schemas/rls/policies/user_data [schemas/auth/tables/users schemas/api/tables/requests] 2024-01-01T00:00:08Z +@v1.0.0 2024-01-01T00:00:09Z +``` + +### Grouping (folder-based, depth=1) + +| Change | Package | +|--------|---------| +| schemas/public | core | +| extensions/uuid | core | +| schemas/auth/types/user_role | auth | +| schemas/auth/tables/users | auth | +| schemas/auth/tables/sessions | auth | +| schemas/auth/functions/login | auth | +| schemas/api/tables/requests | api | +| schemas/api/functions/handle | api | +| schemas/rls/policies/user_data | rls | + +### Package Dependencies + +``` +core: [] +auth: [core] +api: [core, auth] +rls: [auth, api] +``` + +### Output: 4 Packages + +**core/pgpm.plan**: +``` +%project=core +%uri=core + +schemas/public 2024-01-01T00:00:00Z +extensions/uuid [schemas/public] 2024-01-01T00:00:01Z +@v1.0.0 2024-01-01T00:00:09Z +``` + +**auth/pgpm.plan**: +``` +%project=auth +%uri=auth + +schemas/auth/types/user_role [core:@v1.0.0] 2024-01-01T00:00:02Z +schemas/auth/tables/users [core:extensions/uuid schemas/auth/types/user_role] 2024-01-01T00:00:03Z +schemas/auth/tables/sessions [schemas/auth/tables/users] 2024-01-01T00:00:04Z +schemas/auth/functions/login [schemas/auth/tables/users schemas/auth/tables/sessions] 2024-01-01T00:00:05Z +@v1.0.0 2024-01-01T00:00:09Z +``` + +**api/pgpm.plan**: +``` +%project=api +%uri=api + +schemas/api/tables/requests [auth:schemas/auth/tables/users] 2024-01-01T00:00:06Z +schemas/api/functions/handle [schemas/api/tables/requests] 2024-01-01T00:00:07Z +@v1.0.0 2024-01-01T00:00:09Z +``` + +**rls/pgpm.plan**: +``` +%project=rls +%uri=rls + +schemas/rls/policies/user_data [auth:schemas/auth/tables/users api:schemas/api/tables/requests] 2024-01-01T00:00:08Z +@v1.0.0 2024-01-01T00:00:09Z +``` + +### Workspace Manifest (pgpm.json) + +```json +{ + "packages": ["packages/*"], + "slicing": { + "deployOrder": ["core", "auth", "api", "rls"], + "dependencies": { + "core": [], + "auth": ["core"], + "api": ["core", "auth"], + "rls": ["auth", "api"] + } + } +} +``` + +### Statistics + +``` +Total changes: 9 +Packages created: 4 +Internal edges: 5 +Cross-package edges: 4 +Cross-package ratio: 44% +``` + +## CLI Interface Proposal + +```bash +# Basic usage +pgpm export --slice --strategy folder + +# With options +pgpm export --slice \ + --strategy folder \ + --depth 2 \ + --output ./packages \ + --use-tags \ + --min-changes 5 + +# Community detection +pgpm export --slice \ + --strategy community \ + --max-packages 10 + +# Explicit mapping +pgpm export --slice \ + --strategy explicit \ + --mapping ./slice-config.json + +# Dry run (show what would be created) +pgpm export --slice --dry-run + +# Validate existing slicing +pgpm slice validate +``` + +## Integration with Existing PGPM + +### New Files + +1. `pgpm/core/src/slice/index.ts` - Main slicing logic +2. `pgpm/core/src/slice/strategies.ts` - Grouping strategies +3. `pgpm/core/src/slice/validation.ts` - Validation functions +4. `pgpm/core/src/slice/output.ts` - Output generation +5. `pgpm/cli/src/commands/slice.ts` - CLI command + +### Modified Files + +1. `pgpm/core/src/export/export-migrations.ts` - Add `--slice` option +2. `pgpm/types/src/pgpm.ts` - Add `SlicingConfig` to workspace config + +### Reused Components + +- `parsePlanFile()` from `files/plan/parser.ts` +- `writePlanFile()` from `files/plan/writer.ts` +- `PgpmPackage` class for module creation +- `resolveDependencies()` for validation + +## Future Enhancements + +1. **Interactive mode**: Let users adjust groupings before generating +2. **Visualization**: Generate dependency graph visualization +3. **Incremental slicing**: Add new changes to existing sliced packages +4. **Merge packages**: Combine packages that have grown too interdependent +5. **Split packages**: Further divide packages that have grown too large +6. **Migration path**: Generate scripts to migrate existing deployments diff --git a/pgpm/cli/src/commands.ts b/pgpm/cli/src/commands.ts index 8211d5300..72c5d05b6 100644 --- a/pgpm/cli/src/commands.ts +++ b/pgpm/cli/src/commands.ts @@ -26,6 +26,7 @@ import upgrade from './commands/upgrade'; import remove from './commands/remove'; import renameCmd from './commands/rename'; import revert from './commands/revert'; +import slice from './commands/slice'; import tag from './commands/tag'; import testPackages from './commands/test-packages'; import verify from './commands/verify'; @@ -64,12 +65,13 @@ export const createPgpmCommandMap = (skipPgTeardown: boolean = false): Record Path to source plan file (default: pgpm.plan in current module) + --output Output directory for sliced packages (default: ./sliced) + --depth Folder depth for package extraction (default: 1) + --prefix Prefix to strip from paths (default: schemas) + --default Default package name for unmatched changes (default: core) + --min-changes Minimum changes per package (smaller packages are merged) + --use-tags Use tags for cross-package dependencies + --dry-run Show what would be created without writing files + --overwrite Overwrite existing package directories + --copy-files Copy SQL files from source to output packages + --cwd Working directory (default: current directory) + +Examples: + pgpm slice Slice current module's plan + pgpm slice --dry-run Preview slicing without writing files + pgpm slice --depth 2 Use 2-level folder grouping + pgpm slice --output ./packages Output to specific directory + pgpm slice --min-changes 10 Merge packages with fewer than 10 changes +`; + +export default async ( + argv: Partial>, + prompter: Inquirerer, + _options: CLIOptions +) => { + // Show usage if explicitly requested + if (argv.help || argv.h) { + console.log(sliceUsageText); + process.exit(0); + } + + const { username, email } = getGitConfigInfo(); + const cwd = argv.cwd ?? process.cwd(); + const project = new PgpmPackage(cwd); + + // Determine source plan file + let sourcePlan: string; + if (argv.plan) { + sourcePlan = resolve(cwd, argv.plan); + } else if (project.isInModule()) { + sourcePlan = resolve(project.getModulePath()!, 'pgpm.plan'); + } else { + // Prompt for plan file + const { planPath } = await prompter.prompt(argv, [ + { + type: 'text', + name: 'planPath', + message: 'Path to source plan file', + default: 'pgpm.plan', + required: true + } + ]); + sourcePlan = resolve(cwd, planPath); + } + + // Determine output directory + const outputDir = argv.output + ? resolve(cwd, argv.output) + : resolve(cwd, 'sliced'); + + // Get configuration options + const { depth, prefix, defaultPackage, minChanges, useTags } = await prompter.prompt(argv, [ + { + type: 'number', + name: 'depth', + message: 'Folder depth for package extraction', + default: 1, + useDefault: true + }, + { + type: 'text', + name: 'prefix', + message: 'Prefix to strip from paths', + default: 'schemas', + useDefault: true + }, + { + type: 'text', + name: 'defaultPackage', + message: 'Default package name for unmatched changes', + default: 'core', + useDefault: true + }, + { + type: 'number', + name: 'minChanges', + message: 'Minimum changes per package (0 to disable merging)', + default: 0, + useDefault: true + }, + { + type: 'confirm', + name: 'useTags', + message: 'Use tags for cross-package dependencies?', + default: false, + useDefault: true + } + ]); + + // Build slice configuration + const config: SliceConfig = { + sourcePlan, + outputDir, + strategy: { + type: 'folder', + depth: argv.depth ?? depth ?? 1, + prefixToStrip: argv.prefix ?? prefix ?? 'schemas' + }, + defaultPackage: argv.default ?? defaultPackage ?? 'core', + minChangesPerPackage: argv['min-changes'] ?? minChanges ?? 0, + useTagsForCrossPackageDeps: argv['use-tags'] ?? useTags ?? false, + author: `${username} <${email}>` + }; + + console.log(`\nSlicing plan: ${sourcePlan}`); + console.log(`Output directory: ${outputDir}`); + console.log(`Strategy: folder-based (depth=${config.strategy.type === 'folder' ? config.strategy.depth : 1})`); + console.log(''); + + // Perform slicing + const result = slicePlan(config); + + // Handle dry run + if (argv['dry-run'] || argv.dryRun) { + const report = generateDryRunReport(result); + console.log(report); + prompter.close(); + return argv; + } + + // Show summary before writing + console.log(`Found ${result.stats.totalChanges} changes`); + console.log(`Creating ${result.stats.packagesCreated} packages`); + console.log(`Cross-package dependency ratio: ${(result.stats.crossPackageRatio * 100).toFixed(1)}%`); + console.log(''); + + // Show warnings + if (result.warnings.length > 0) { + console.log('Warnings:'); + for (const warning of result.warnings) { + console.log(` [${warning.type}] ${warning.message}`); + } + console.log(''); + } + + // Show deploy order + console.log('Deploy order:'); + for (let i = 0; i < result.workspace.deployOrder.length; i++) { + const pkg = result.workspace.deployOrder[i]; + const deps = result.workspace.dependencies[pkg] || []; + const depStr = deps.length > 0 ? ` -> ${deps.join(', ')}` : ''; + console.log(` ${i + 1}. ${pkg}${depStr}`); + } + console.log(''); + + // Confirm before writing (unless --overwrite is specified) + if (!argv.overwrite) { + const confirmResult = await prompter.prompt({} as Record, [ + { + type: 'confirm', + name: 'confirm', + message: 'Proceed with writing packages?', + default: true + } + ]) as { confirm: boolean }; + + if (!confirmResult.confirm) { + console.log('Aborted.'); + prompter.close(); + return argv; + } + } + + // Determine source directory for copying files + let sourceDir: string | undefined; + if (argv['copy-files'] || argv.copyFiles) { + if (project.isInModule()) { + sourceDir = project.getModulePath(); + } else { + // Use directory containing the plan file + sourceDir = resolve(sourcePlan, '..'); + } + } + + // Write packages to disk + writeSliceResult(result, { + outputDir, + overwrite: argv.overwrite ?? false, + copySourceFiles: argv['copy-files'] ?? argv.copyFiles ?? false, + sourceDir + }); + + prompter.close(); + + console.log(` + ||| + (o o) + ooO--(_)--Ooo- + +Sliced into ${result.stats.packagesCreated} packages! + +Output: ${outputDir} +`); + + return argv; +}; diff --git a/pgpm/cli/src/index.ts b/pgpm/cli/src/index.ts index 9b98f735d..4c3711372 100644 --- a/pgpm/cli/src/index.ts +++ b/pgpm/cli/src/index.ts @@ -25,6 +25,7 @@ export { default as plan } from './commands/plan'; export { default as remove } from './commands/remove'; export { default as renameCmd } from './commands/rename'; export { default as revert } from './commands/revert'; +export { default as slice } from './commands/slice'; export { default as tag } from './commands/tag'; export { default as testPackages } from './commands/test-packages'; export { default as verify } from './commands/verify'; diff --git a/pgpm/core/__tests__/slice/slice.test.ts b/pgpm/core/__tests__/slice/slice.test.ts new file mode 100644 index 000000000..2bbc6ce38 --- /dev/null +++ b/pgpm/core/__tests__/slice/slice.test.ts @@ -0,0 +1,302 @@ +import { mkdirSync, rmSync, writeFileSync } from 'fs'; +import { join } from 'path'; + +import { + buildDependencyGraph, + validateDAG, + extractPackageFromPath, + assignChangesToPackages, + buildPackageDependencies, + detectPackageCycle, + computeDeployOrder, + topologicalSortWithinPackage, + slicePlan +} from '../../src/slice'; +import { ExtendedPlanFile } from '../../src/files/types'; + +describe('Slice Module', () => { + const testDir = join(__dirname, 'test-slice'); + + beforeAll(() => { + mkdirSync(testDir, { recursive: true }); + }); + + afterAll(() => { + rmSync(testDir, { recursive: true, force: true }); + }); + + describe('extractPackageFromPath', () => { + it('should extract package name from schemas path', () => { + expect(extractPackageFromPath('schemas/auth/tables/users')).toBe('auth'); + expect(extractPackageFromPath('schemas/public/functions/get_user')).toBe('public'); + expect(extractPackageFromPath('schemas/rls/policies/user_policy')).toBe('rls'); + }); + + it('should handle extensions folder as core', () => { + expect(extractPackageFromPath('extensions/uuid-ossp')).toBe('core'); + expect(extractPackageFromPath('extensions/pgcrypto')).toBe('core'); + }); + + it('should handle migrate folder as core', () => { + expect(extractPackageFromPath('migrate/init')).toBe('core'); + }); + + it('should handle custom depth', () => { + expect(extractPackageFromPath('schemas/auth/tables/users', 2)).toBe('auth/tables'); + }); + + it('should handle custom prefix', () => { + expect(extractPackageFromPath('custom/auth/tables/users', 1, 'custom')).toBe('auth'); + }); + + it('should return core for paths without schema prefix', () => { + expect(extractPackageFromPath('init')).toBe('init'); + }); + }); + + describe('buildDependencyGraph', () => { + it('should build graph from plan file', () => { + const plan: ExtendedPlanFile = { + package: 'test', + uri: '', + changes: [ + { name: 'schemas/auth/tables/users', dependencies: [] }, + { name: 'schemas/auth/tables/posts', dependencies: ['schemas/auth/tables/users'] }, + { name: 'schemas/public/functions/get_user', dependencies: ['schemas/auth/tables/users'] } + ], + tags: [] + }; + + const graph = buildDependencyGraph(plan); + + expect(graph.nodes.size).toBe(3); + expect(graph.edges.get('schemas/auth/tables/posts')?.has('schemas/auth/tables/users')).toBe(true); + expect(graph.reverseEdges.get('schemas/auth/tables/users')?.has('schemas/auth/tables/posts')).toBe(true); + }); + }); + + describe('validateDAG', () => { + it('should pass for valid DAG', () => { + const plan: ExtendedPlanFile = { + package: 'test', + uri: '', + changes: [ + { name: 'a', dependencies: [] }, + { name: 'b', dependencies: ['a'] }, + { name: 'c', dependencies: ['b'] } + ], + tags: [] + }; + + const graph = buildDependencyGraph(plan); + expect(() => validateDAG(graph)).not.toThrow(); + }); + + it('should throw for cycle', () => { + const plan: ExtendedPlanFile = { + package: 'test', + uri: '', + changes: [ + { name: 'a', dependencies: ['c'] }, + { name: 'b', dependencies: ['a'] }, + { name: 'c', dependencies: ['b'] } + ], + tags: [] + }; + + const graph = buildDependencyGraph(plan); + expect(() => validateDAG(graph)).toThrow(/Cycle detected/); + }); + }); + + describe('assignChangesToPackages', () => { + it('should assign changes using folder strategy', () => { + const plan: ExtendedPlanFile = { + package: 'test', + uri: '', + changes: [ + { name: 'schemas/auth/tables/users', dependencies: [] }, + { name: 'schemas/auth/tables/posts', dependencies: [] }, + { name: 'schemas/public/functions/get_user', dependencies: [] }, + { name: 'extensions/uuid-ossp', dependencies: [] } + ], + tags: [] + }; + + const graph = buildDependencyGraph(plan); + const assignments = assignChangesToPackages(graph, { type: 'folder' }); + + expect(assignments.get('auth')?.size).toBe(2); + expect(assignments.get('public')?.size).toBe(1); + expect(assignments.get('core')?.size).toBe(1); + }); + + it('should use explicit mapping strategy', () => { + const plan: ExtendedPlanFile = { + package: 'test', + uri: '', + changes: [ + { name: 'change_a', dependencies: [] }, + { name: 'change_b', dependencies: [] }, + { name: 'change_c', dependencies: [] } + ], + tags: [] + }; + + const graph = buildDependencyGraph(plan); + const assignments = assignChangesToPackages(graph, { + type: 'explicit', + mapping: { + 'change_a': 'pkg1', + 'change_b': 'pkg1', + 'change_c': 'pkg2' + } + }); + + expect(assignments.get('pkg1')?.size).toBe(2); + expect(assignments.get('pkg2')?.size).toBe(1); + }); + }); + + describe('buildPackageDependencies', () => { + it('should detect cross-package dependencies', () => { + const plan: ExtendedPlanFile = { + package: 'test', + uri: '', + changes: [ + { name: 'schemas/auth/tables/users', dependencies: [] }, + { name: 'schemas/public/functions/get_user', dependencies: ['schemas/auth/tables/users'] } + ], + tags: [] + }; + + const graph = buildDependencyGraph(plan); + const assignments = assignChangesToPackages(graph, { type: 'folder' }); + const pkgDeps = buildPackageDependencies(graph, assignments); + + expect(pkgDeps.get('public')?.has('auth')).toBe(true); + expect(pkgDeps.get('auth')?.size).toBe(0); + }); + }); + + describe('detectPackageCycle', () => { + it('should return null for acyclic package deps', () => { + const deps = new Map>(); + deps.set('auth', new Set()); + deps.set('public', new Set(['auth'])); + deps.set('rls', new Set(['auth', 'public'])); + + expect(detectPackageCycle(deps)).toBeNull(); + }); + + it('should detect package cycle', () => { + const deps = new Map>(); + deps.set('auth', new Set(['rls'])); + deps.set('public', new Set(['auth'])); + deps.set('rls', new Set(['public'])); + + const cycle = detectPackageCycle(deps); + expect(cycle).not.toBeNull(); + expect(cycle!.length).toBeGreaterThan(2); + }); + }); + + describe('computeDeployOrder', () => { + it('should compute correct deploy order', () => { + const deps = new Map>(); + deps.set('auth', new Set()); + deps.set('public', new Set(['auth'])); + deps.set('rls', new Set(['auth', 'public'])); + + const order = computeDeployOrder(deps); + + expect(order.indexOf('auth')).toBeLessThan(order.indexOf('public')); + expect(order.indexOf('public')).toBeLessThan(order.indexOf('rls')); + }); + }); + + describe('topologicalSortWithinPackage', () => { + it('should sort changes within package', () => { + const plan: ExtendedPlanFile = { + package: 'test', + uri: '', + changes: [ + { name: 'a', dependencies: [] }, + { name: 'b', dependencies: ['a'] }, + { name: 'c', dependencies: ['b'] } + ], + tags: [] + }; + + const graph = buildDependencyGraph(plan); + const changes = new Set(['a', 'b', 'c']); + const sorted = topologicalSortWithinPackage(changes, graph); + + expect(sorted.indexOf('a')).toBeLessThan(sorted.indexOf('b')); + expect(sorted.indexOf('b')).toBeLessThan(sorted.indexOf('c')); + }); + }); + + describe('slicePlan', () => { + it('should slice a plan file into packages', () => { + const planContent = `%syntax-version=1.0.0 +%project=test-project +%uri=https://github.com/test/project + +extensions/uuid-ossp 2024-01-01T00:00:00Z Developer # UUID extension +schemas/auth/tables/users [extensions/uuid-ossp] 2024-01-02T00:00:00Z Developer # Users table +schemas/auth/tables/sessions [schemas/auth/tables/users] 2024-01-03T00:00:00Z Developer # Sessions table +schemas/public/functions/get_user [schemas/auth/tables/users] 2024-01-04T00:00:00Z Developer # Get user function +schemas/rls/policies/user_policy [schemas/auth/tables/users schemas/public/functions/get_user] 2024-01-05T00:00:00Z Developer # User policy +`; + const planPath = join(testDir, 'slice-test.plan'); + writeFileSync(planPath, planContent); + + const result = slicePlan({ + sourcePlan: planPath, + outputDir: join(testDir, 'output'), + strategy: { type: 'folder' } + }); + + expect(result.stats.totalChanges).toBe(5); + expect(result.stats.packagesCreated).toBeGreaterThan(1); + expect(result.packages.length).toBeGreaterThan(1); + + // Check that packages are created + const packageNames = result.packages.map(p => p.name); + expect(packageNames).toContain('core'); + expect(packageNames).toContain('auth'); + expect(packageNames).toContain('public'); + expect(packageNames).toContain('rls'); + + // Check deploy order + const deployOrder = result.workspace.deployOrder; + expect(deployOrder.indexOf('core')).toBeLessThan(deployOrder.indexOf('auth')); + expect(deployOrder.indexOf('auth')).toBeLessThan(deployOrder.indexOf('public')); + }); + + it('should generate cross-package dependencies', () => { + const planContent = `%syntax-version=1.0.0 +%project=test-project + +schemas/auth/tables/users 2024-01-01T00:00:00Z Developer +schemas/public/functions/get_user [schemas/auth/tables/users] 2024-01-02T00:00:00Z Developer +`; + const planPath = join(testDir, 'cross-deps.plan'); + writeFileSync(planPath, planContent); + + const result = slicePlan({ + sourcePlan: planPath, + outputDir: join(testDir, 'output2'), + strategy: { type: 'folder' } + }); + + const publicPkg = result.packages.find(p => p.name === 'public'); + expect(publicPkg).toBeDefined(); + expect(publicPkg!.packageDependencies).toContain('auth'); + + // Check that the plan content has cross-package reference + expect(publicPkg!.planContent).toContain('auth:schemas/auth/tables/users'); + }); + }); +}); diff --git a/pgpm/core/src/index.ts b/pgpm/core/src/index.ts index 1010463fc..fb507c156 100644 --- a/pgpm/core/src/index.ts +++ b/pgpm/core/src/index.ts @@ -1,6 +1,7 @@ export * from './core/class/pgpm'; export * from './export/export-meta'; export * from './export/export-migrations'; +export * from './slice'; export * from './extensions/extensions'; export * from './modules/modules'; export * from './packaging/package'; diff --git a/pgpm/core/src/slice/index.ts b/pgpm/core/src/slice/index.ts new file mode 100644 index 000000000..2b87d9157 --- /dev/null +++ b/pgpm/core/src/slice/index.ts @@ -0,0 +1,3 @@ +export * from './types'; +export * from './slice'; +export * from './output'; diff --git a/pgpm/core/src/slice/output.ts b/pgpm/core/src/slice/output.ts new file mode 100644 index 000000000..008b1004d --- /dev/null +++ b/pgpm/core/src/slice/output.ts @@ -0,0 +1,174 @@ +import fs from 'fs'; +import path from 'path'; + +import { SliceResult, PackageOutput } from './types'; + +/** + * Options for writing sliced packages to disk + */ +export interface WriteSliceOptions { + /** Base output directory */ + outputDir: string; + + /** Whether to overwrite existing packages */ + overwrite?: boolean; + + /** Whether to copy SQL files from source */ + copySourceFiles?: boolean; + + /** Source directory containing deploy/revert/verify folders */ + sourceDir?: string; +} + +/** + * Write sliced packages to disk + */ +export function writeSliceResult( + result: SliceResult, + options: WriteSliceOptions +): void { + const { outputDir, overwrite = false, copySourceFiles = false, sourceDir } = options; + + // Create output directory + fs.mkdirSync(outputDir, { recursive: true }); + + // Write each package + for (const pkg of result.packages) { + writePackage(pkg, outputDir, overwrite, copySourceFiles, sourceDir); + } + + // Write workspace manifest + const manifestPath = path.join(outputDir, 'pgpm-workspace.json'); + const manifestContent = JSON.stringify( + { + packages: result.workspace.packages.map(p => `packages/${p}`), + slicing: { + deployOrder: result.workspace.deployOrder, + dependencies: result.workspace.dependencies + }, + stats: result.stats + }, + null, + 2 + ); + fs.writeFileSync(manifestPath, manifestContent); +} + +/** + * Write a single package to disk + */ +function writePackage( + pkg: PackageOutput, + outputDir: string, + overwrite: boolean, + copySourceFiles: boolean, + sourceDir?: string +): void { + const pkgDir = path.join(outputDir, 'packages', pkg.name); + + // Check if package already exists + if (fs.existsSync(pkgDir) && !overwrite) { + throw new Error(`Package directory already exists: ${pkgDir}. Use --overwrite to replace.`); + } + + // Create package directory structure + fs.mkdirSync(pkgDir, { recursive: true }); + fs.mkdirSync(path.join(pkgDir, 'deploy'), { recursive: true }); + fs.mkdirSync(path.join(pkgDir, 'revert'), { recursive: true }); + fs.mkdirSync(path.join(pkgDir, 'verify'), { recursive: true }); + + // Write plan file + fs.writeFileSync(path.join(pkgDir, 'pgpm.plan'), pkg.planContent); + + // Write control file + fs.writeFileSync(path.join(pkgDir, `${pkg.name}.control`), pkg.controlContent); + + // Copy SQL files if requested + if (copySourceFiles && sourceDir) { + for (const change of pkg.changes) { + copyChangeFiles(change.name, sourceDir, pkgDir); + } + } +} + +/** + * Copy deploy/revert/verify SQL files for a change + */ +function copyChangeFiles( + changeName: string, + sourceDir: string, + targetDir: string +): void { + const types = ['deploy', 'revert', 'verify']; + + for (const type of types) { + const sourceFile = path.join(sourceDir, type, `${changeName}.sql`); + const targetFile = path.join(targetDir, type, `${changeName}.sql`); + + if (fs.existsSync(sourceFile)) { + // Create target directory if needed + const targetSubDir = path.dirname(targetFile); + fs.mkdirSync(targetSubDir, { recursive: true }); + + // Copy file + fs.copyFileSync(sourceFile, targetFile); + } + } +} + +/** + * Generate a dry-run report of what would be created + */ +export function generateDryRunReport(result: SliceResult): string { + const lines: string[] = []; + + lines.push('=== PGPM Slice Dry Run Report ===\n'); + + // Statistics + lines.push('Statistics:'); + lines.push(` Total changes: ${result.stats.totalChanges}`); + lines.push(` Packages to create: ${result.stats.packagesCreated}`); + lines.push(` Internal edges: ${result.stats.internalEdges}`); + lines.push(` Cross-package edges: ${result.stats.crossPackageEdges}`); + lines.push(` Cross-package ratio: ${(result.stats.crossPackageRatio * 100).toFixed(1)}%`); + lines.push(''); + + // Warnings + if (result.warnings.length > 0) { + lines.push('Warnings:'); + for (const warning of result.warnings) { + lines.push(` [${warning.type}] ${warning.message}`); + if (warning.suggestedAction) { + lines.push(` Suggestion: ${warning.suggestedAction}`); + } + } + lines.push(''); + } + + // Deploy order + lines.push('Deploy Order:'); + for (let i = 0; i < result.workspace.deployOrder.length; i++) { + const pkg = result.workspace.deployOrder[i]; + const deps = result.workspace.dependencies[pkg] || []; + const depStr = deps.length > 0 ? ` (depends on: ${deps.join(', ')})` : ''; + lines.push(` ${i + 1}. ${pkg}${depStr}`); + } + lines.push(''); + + // Package details + lines.push('Packages:'); + for (const pkg of result.packages) { + lines.push(`\n ${pkg.name}/`); + lines.push(` Changes: ${pkg.changes.length}`); + lines.push(` Dependencies: ${pkg.packageDependencies.join(', ') || 'none'}`); + lines.push(' Contents:'); + for (const change of pkg.changes.slice(0, 5)) { + lines.push(` - ${change.name}`); + } + if (pkg.changes.length > 5) { + lines.push(` ... and ${pkg.changes.length - 5} more`); + } + } + + return lines.join('\n'); +} diff --git a/pgpm/core/src/slice/slice.ts b/pgpm/core/src/slice/slice.ts new file mode 100644 index 000000000..502b719ba --- /dev/null +++ b/pgpm/core/src/slice/slice.ts @@ -0,0 +1,637 @@ +import { Change, Tag, ExtendedPlanFile } from '../files/types'; +import { parsePlanFile } from '../files/plan/parser'; +import { + SliceConfig, + SliceResult, + DependencyGraph, + PackageOutput, + WorkspaceManifest, + SliceWarning, + SliceStats, + GroupingStrategy +} from './types'; + +/** + * Build a dependency graph from a parsed plan file + */ +export function buildDependencyGraph(plan: ExtendedPlanFile): DependencyGraph { + const graph: DependencyGraph = { + nodes: new Map(), + edges: new Map(), + reverseEdges: new Map(), + tags: new Map(), + plan + }; + + // Add all changes as nodes + for (const change of plan.changes) { + graph.nodes.set(change.name, change); + graph.edges.set(change.name, new Set(change.dependencies || [])); + + // Build reverse edges for dependency analysis + for (const dep of change.dependencies || []) { + if (!graph.reverseEdges.has(dep)) { + graph.reverseEdges.set(dep, new Set()); + } + graph.reverseEdges.get(dep)!.add(change.name); + } + } + + // Index tags by change + for (const tag of plan.tags || []) { + if (!graph.tags.has(tag.change)) { + graph.tags.set(tag.change, []); + } + graph.tags.get(tag.change)!.push(tag); + } + + return graph; +} + +/** + * Validate that the dependency graph is a DAG (no cycles) + */ +export function validateDAG(graph: DependencyGraph): void { + const visited = new Set(); + const visiting = new Set(); + + function dfs(node: string, path: string[]): void { + if (visiting.has(node)) { + throw new Error(`Cycle detected: ${[...path, node].join(' -> ')}`); + } + if (visited.has(node)) return; + + visiting.add(node); + const deps = graph.edges.get(node) || new Set(); + for (const dep of deps) { + // Only follow internal dependencies + if (graph.nodes.has(dep)) { + dfs(dep, [...path, node]); + } + } + visiting.delete(node); + visited.add(node); + } + + for (const node of graph.nodes.keys()) { + dfs(node, []); + } +} + +/** + * Extract package name from a change path using folder-based strategy + */ +export function extractPackageFromPath( + changeName: string, + depth: number = 1, + prefixToStrip: string = 'schemas' +): string { + const parts = changeName.split('/'); + + // Handle special folders that should go to core + if (parts[0] === 'extensions' || parts[0] === 'migrate') { + return 'core'; + } + + // Strip prefix if present + let startIdx = 0; + if (parts[0] === prefixToStrip) { + startIdx = 1; + } + + // Get package name at specified depth + if (parts.length > startIdx) { + // For depth > 1, join multiple segments + const endIdx = Math.min(startIdx + depth, parts.length - 1); + if (endIdx > startIdx) { + return parts.slice(startIdx, endIdx).join('/'); + } + return parts[startIdx]; + } + + return 'core'; +} + +/** + * Assign changes to packages based on grouping strategy + */ +export function assignChangesToPackages( + graph: DependencyGraph, + strategy: GroupingStrategy, + defaultPackage: string = 'core' +): Map> { + const assignments = new Map>(); + + for (const [changeName] of graph.nodes) { + let packageName: string; + + switch (strategy.type) { + case 'folder': { + const depth = strategy.depth ?? 1; + const prefix = strategy.prefixToStrip ?? 'schemas'; + packageName = extractPackageFromPath(changeName, depth, prefix); + break; + } + case 'explicit': { + packageName = strategy.mapping[changeName] || defaultPackage; + break; + } + default: + packageName = defaultPackage; + } + + // Fallback to default package + if (!packageName) { + packageName = defaultPackage; + } + + if (!assignments.has(packageName)) { + assignments.set(packageName, new Set()); + } + assignments.get(packageName)!.add(changeName); + } + + return assignments; +} + +/** + * Merge small packages into larger ones + */ +export function mergeSmallPackages( + assignments: Map>, + minChanges: number, + defaultPackage: string = 'core' +): Map> { + const result = new Map>(); + + // Ensure default package exists + if (!result.has(defaultPackage)) { + result.set(defaultPackage, new Set()); + } + + for (const [pkg, changes] of assignments) { + if (changes.size < minChanges && pkg !== defaultPackage) { + // Merge into default package + const defaultChanges = result.get(defaultPackage)!; + for (const change of changes) { + defaultChanges.add(change); + } + } else { + result.set(pkg, new Set(changes)); + } + } + + // Remove empty default package if it exists + if (result.get(defaultPackage)?.size === 0) { + result.delete(defaultPackage); + } + + return result; +} + +/** + * Build package-level dependency graph + */ +export function buildPackageDependencies( + graph: DependencyGraph, + assignments: Map> +): Map> { + const packageDeps = new Map>(); + + // Build reverse lookup: change -> package + const changeToPackage = new Map(); + for (const [pkg, changes] of assignments) { + packageDeps.set(pkg, new Set()); + for (const change of changes) { + changeToPackage.set(change, pkg); + } + } + + // Check each change's dependencies + for (const [changeName, deps] of graph.edges) { + const myPackage = changeToPackage.get(changeName); + if (!myPackage) continue; + + for (const dep of deps) { + const depPackage = changeToPackage.get(dep); + + if (depPackage && depPackage !== myPackage) { + // Cross-package dependency + packageDeps.get(myPackage)!.add(depPackage); + } + } + } + + return packageDeps; +} + +/** + * Detect cycles in package dependency graph + */ +export function detectPackageCycle(deps: Map>): string[] | null { + const visited = new Set(); + const visiting = new Set(); + + function dfs(pkg: string, path: string[]): string[] | null { + if (visiting.has(pkg)) { + return [...path, pkg]; + } + if (visited.has(pkg)) return null; + + visiting.add(pkg); + const pkgDeps = deps.get(pkg) || new Set(); + for (const dep of pkgDeps) { + const cycle = dfs(dep, [...path, pkg]); + if (cycle) return cycle; + } + visiting.delete(pkg); + visited.add(pkg); + return null; + } + + for (const pkg of deps.keys()) { + const cycle = dfs(pkg, []); + if (cycle) return cycle; + } + return null; +} + +/** + * Compute deployment order for packages (topological sort) + */ +export function computeDeployOrder(packageDeps: Map>): string[] { + const order: string[] = []; + const visited = new Set(); + const allPackages = new Set(); + + // Collect all packages + for (const pkg of packageDeps.keys()) { + allPackages.add(pkg); + } + for (const deps of packageDeps.values()) { + for (const dep of deps) { + allPackages.add(dep); + } + } + + function visit(pkg: string): void { + if (visited.has(pkg)) return; + visited.add(pkg); + + const deps = packageDeps.get(pkg) || new Set(); + for (const dep of deps) { + visit(dep); + } + + order.push(pkg); + } + + // Visit all packages (sorted for determinism) + for (const pkg of [...allPackages].sort()) { + visit(pkg); + } + + return order; +} + +/** + * Topological sort of changes within a package + */ +export function topologicalSortWithinPackage( + changes: Set, + graph: DependencyGraph +): string[] { + const result: string[] = []; + const visited = new Set(); + + function visit(change: string): void { + if (visited.has(change)) return; + if (!changes.has(change)) return; + + visited.add(change); + + const deps = graph.edges.get(change) || new Set(); + for (const dep of deps) { + if (changes.has(dep)) { + visit(dep); + } + } + + result.push(change); + } + + // Visit in alphabetical order for determinism + for (const change of [...changes].sort()) { + visit(change); + } + + return result; +} + +/** + * Generate plan file content for a package + */ +export function generatePlanContent( + pkgName: string, + entries: Change[], + tags: Tag[] = [] +): string { + let content = `%syntax-version=1.0.0\n`; + content += `%project=${pkgName}\n`; + content += `%uri=${pkgName}\n\n`; + + for (const entry of entries) { + let line = entry.name; + + if (entry.dependencies && entry.dependencies.length > 0) { + line += ` [${entry.dependencies.join(' ')}]`; + } + + if (entry.timestamp) { + line += ` ${entry.timestamp}`; + if (entry.planner) { + line += ` ${entry.planner}`; + if (entry.email) { + line += ` <${entry.email}>`; + } + } + } + + if (entry.comment) { + line += ` # ${entry.comment}`; + } + + content += line + '\n'; + + // Add tags associated with this change + const changeTags = tags.filter(t => t.change === entry.name); + for (const tag of changeTags) { + let tagLine = `@${tag.name}`; + if (tag.timestamp) { + tagLine += ` ${tag.timestamp}`; + if (tag.planner) { + tagLine += ` ${tag.planner}`; + if (tag.email) { + tagLine += ` <${tag.email}>`; + } + } + } + if (tag.comment) { + tagLine += ` # ${tag.comment}`; + } + content += tagLine + '\n'; + } + } + + return content; +} + +/** + * Generate control file content for a package + */ +export function generateControlContent( + pkgName: string, + deps: Set +): string { + let content = `# ${pkgName} extension\n`; + content += `comment = '${pkgName} module'\n`; + content += `default_version = '0.0.1'\n`; + content += `relocatable = false\n`; + + if (deps.size > 0) { + content += `requires = '${[...deps].sort().join(', ')}'\n`; + } + + return content; +} + +/** + * Generate a single package output + */ +export function generateSinglePackage( + pkgName: string, + changes: Set, + graph: DependencyGraph, + changeToPackage: Map, + pkgDeps: Set, + useTagsForCrossPackageDeps: boolean = false +): PackageOutput { + // Sort changes in topological order within package + const sortedChanges = topologicalSortWithinPackage(changes, graph); + + // Build plan entries with updated dependencies + const planEntries: Change[] = []; + + for (const changeName of sortedChanges) { + const originalChange = graph.nodes.get(changeName)!; + const deps = graph.edges.get(changeName) || new Set(); + + const newDeps: string[] = []; + + for (const dep of deps) { + const depPkg = changeToPackage.get(dep); + + if (!depPkg) { + // External dependency (from installed module) - keep as-is + newDeps.push(dep); + } else if (depPkg === pkgName) { + // Internal dependency - keep as-is + newDeps.push(dep); + } else { + // Cross-package dependency + if (useTagsForCrossPackageDeps) { + // Find latest tag in the dependency package + const depPkgChanges = [...graph.nodes.keys()] + .filter(c => changeToPackage.get(c) === depPkg); + const lastChange = depPkgChanges[depPkgChanges.length - 1]; + const tags = graph.tags.get(lastChange); + + if (tags && tags.length > 0) { + const latestTag = tags[tags.length - 1]; + newDeps.push(`${depPkg}:@${latestTag.name}`); + } else { + newDeps.push(`${depPkg}:${dep}`); + } + } else { + newDeps.push(`${depPkg}:${dep}`); + } + } + } + + planEntries.push({ + name: changeName, + dependencies: newDeps, + timestamp: originalChange.timestamp, + planner: originalChange.planner, + email: originalChange.email, + comment: originalChange.comment + }); + } + + // Collect tags for this package + const packageTags: Tag[] = []; + for (const changeName of sortedChanges) { + const changeTags = graph.tags.get(changeName) || []; + packageTags.push(...changeTags); + } + + // Generate plan content + const planContent = generatePlanContent(pkgName, planEntries, packageTags); + + // Generate control file content + const controlContent = generateControlContent(pkgName, pkgDeps); + + return { + name: pkgName, + planContent, + controlContent, + changes: planEntries, + packageDependencies: [...pkgDeps] + }; +} + +/** + * Main slicing function + */ +export function slicePlan(config: SliceConfig): SliceResult { + // Parse the source plan file + const planResult = parsePlanFile(config.sourcePlan); + if (!planResult.data) { + const errorMessages = planResult.errors?.map(e => `Line ${e.line}: ${e.message}`).join('\n') || 'Unknown error'; + throw new Error(`Failed to parse plan file: ${errorMessages}`); + } + + const plan = planResult.data; + const warnings: SliceWarning[] = []; + + // Build dependency graph + const graph = buildDependencyGraph(plan); + + // Validate DAG + try { + validateDAG(graph); + } catch (error) { + throw new Error(`Invalid plan file: ${(error as Error).message}`); + } + + // Assign changes to packages + let assignments = assignChangesToPackages( + graph, + config.strategy, + config.defaultPackage || 'core' + ); + + // Merge small packages if configured + if (config.minChangesPerPackage && config.minChangesPerPackage > 1) { + assignments = mergeSmallPackages( + assignments, + config.minChangesPerPackage, + config.defaultPackage || 'core' + ); + } + + // Build package dependencies + const packageDeps = buildPackageDependencies(graph, assignments); + + // Check for package cycles + const cycle = detectPackageCycle(packageDeps); + if (cycle) { + warnings.push({ + type: 'cycle_detected', + message: `Package cycle detected: ${cycle.join(' -> ')}`, + suggestedAction: 'Consider merging these packages or reorganizing dependencies' + }); + // For now, we'll proceed but warn - in production we might want to auto-merge + } + + // Build reverse lookup + const changeToPackage = new Map(); + for (const [pkg, changes] of assignments) { + for (const change of changes) { + changeToPackage.set(change, pkg); + } + } + + // Compute deploy order + const deployOrder = computeDeployOrder(packageDeps); + + // Generate packages + const packages: PackageOutput[] = []; + + for (const pkgName of deployOrder) { + const changes = assignments.get(pkgName); + if (!changes || changes.size === 0) continue; + + const pkgOutput = generateSinglePackage( + pkgName, + changes, + graph, + changeToPackage, + packageDeps.get(pkgName) || new Set(), + config.useTagsForCrossPackageDeps || false + ); + + packages.push(pkgOutput); + + // Check for heavy cross-package dependencies + let crossDeps = 0; + let totalDeps = 0; + + for (const change of changes) { + const deps = graph.edges.get(change) || new Set(); + for (const dep of deps) { + totalDeps++; + const depPkg = changeToPackage.get(dep); + if (depPkg && depPkg !== pkgName) { + crossDeps++; + } + } + } + + const ratio = totalDeps > 0 ? crossDeps / totalDeps : 0; + if (ratio > 0.5) { + warnings.push({ + type: 'heavy_cross_deps', + message: `Package "${pkgName}" has ${Math.round(ratio * 100)}% cross-package dependencies`, + suggestedAction: 'Consider merging with dependent packages or reorganizing' + }); + } + } + + // Calculate stats + let internalEdges = 0; + let crossPackageEdges = 0; + + for (const [change, deps] of graph.edges) { + const myPkg = changeToPackage.get(change); + for (const dep of deps) { + const depPkg = changeToPackage.get(dep); + if (depPkg === myPkg) { + internalEdges++; + } else if (depPkg) { + crossPackageEdges++; + } + } + } + + const totalEdges = internalEdges + crossPackageEdges; + + return { + packages, + workspace: { + packages: deployOrder, + deployOrder, + dependencies: Object.fromEntries( + [...packageDeps.entries()].map(([k, v]) => [k, [...v].sort()]) + ) + }, + warnings, + stats: { + totalChanges: graph.nodes.size, + packagesCreated: packages.length, + internalEdges, + crossPackageEdges, + crossPackageRatio: totalEdges > 0 ? crossPackageEdges / totalEdges : 0 + } + }; +} diff --git a/pgpm/core/src/slice/types.ts b/pgpm/core/src/slice/types.ts new file mode 100644 index 000000000..f1ad991fa --- /dev/null +++ b/pgpm/core/src/slice/types.ts @@ -0,0 +1,145 @@ +import { Change, Tag, ExtendedPlanFile } from '../files/types'; + +/** + * Configuration for slicing a plan into multiple packages + */ +export interface SliceConfig { + /** Source plan file path */ + sourcePlan: string; + + /** Output directory for packages */ + outputDir: string; + + /** Grouping strategy */ + strategy: GroupingStrategy; + + /** Package for changes that don't match any group */ + defaultPackage?: string; + + /** Whether to use tags for cross-package deps */ + useTagsForCrossPackageDeps?: boolean; + + /** Minimum changes per package (merge smaller groups) */ + minChangesPerPackage?: number; + + /** Author for generated plan files */ + author?: string; +} + +/** + * Grouping strategy for slicing + */ +export type GroupingStrategy = + | FolderStrategy + | ExplicitStrategy; + +export interface FolderStrategy { + type: 'folder'; + /** Depth in path to extract package name (default: 1) */ + depth?: number; + /** Prefix to strip from paths (default: 'schemas') */ + prefixToStrip?: string; +} + +export interface ExplicitStrategy { + type: 'explicit'; + /** Mapping of change name to package name */ + mapping: Record; +} + +/** + * Dependency graph representation + */ +export interface DependencyGraph { + /** Map of change name to Change object */ + nodes: Map; + /** Map of change name to its dependencies */ + edges: Map>; + /** Map of change name to changes that depend on it */ + reverseEdges: Map>; + /** Map of change name to its tags */ + tags: Map; + /** Original plan metadata */ + plan: ExtendedPlanFile; +} + +/** + * Result of slicing operation + */ +export interface SliceResult { + /** Generated packages */ + packages: PackageOutput[]; + + /** Workspace manifest */ + workspace: WorkspaceManifest; + + /** Warnings/issues encountered */ + warnings: SliceWarning[]; + + /** Statistics */ + stats: SliceStats; +} + +/** + * Output for a single package + */ +export interface PackageOutput { + /** Package name */ + name: string; + + /** Plan file content */ + planContent: string; + + /** Control file content */ + controlContent: string; + + /** Changes in this package */ + changes: Change[]; + + /** Dependencies on other packages */ + packageDependencies: string[]; +} + +/** + * Workspace manifest describing package relationships + */ +export interface WorkspaceManifest { + /** List of package names */ + packages: string[]; + + /** Order in which packages should be deployed */ + deployOrder: string[]; + + /** Map of package name to its package dependencies */ + dependencies: Record; +} + +/** + * Warning generated during slicing + */ +export interface SliceWarning { + type: 'heavy_cross_deps' | 'cycle_detected' | 'orphan_change' | 'merge_required'; + message: string; + affectedChanges?: string[]; + suggestedAction?: string; +} + +/** + * Statistics about the slicing operation + */ +export interface SliceStats { + /** Total number of changes processed */ + totalChanges: number; + + /** Number of packages created */ + packagesCreated: number; + + /** Number of internal dependency edges */ + internalEdges: number; + + /** Number of cross-package dependency edges */ + crossPackageEdges: number; + + /** Ratio of cross-package to total edges */ + crossPackageRatio: number; +}