diff --git a/.gitignore b/.gitignore index 9885b66..46d9564 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .vscode tmp /ripoff +/ripoff-export .DS_Store /export diff --git a/README.md b/README.md index baf6bc7..b6c7d10 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,15 @@ Currently, it attempts to export all data from all tables into a single ripoff f ripoff-export --exclude users --exclude audit_logs /path/to/export ``` +You can also use the `--ignore-on-update` flag to mark columns that should be ignored during subsequent imports (useful for timestamps that shouldn't change): + +```bash +# Export all data but ignore created_at and updated_at columns during updates +ripoff-export --ignore-on-update created_at --ignore-on-update updated_at /path/to/export +``` + +This adds a `~ignore_on_update` metadata field to rows containing the specified columns. During import, these columns will be included for new rows but ignored when updating existing rows, preventing timestamp-only changes from triggering unnecessary updates. + In the future, additional flags may be added to allow you to include tables, add arbitrary `WHERE` conditions, modify the row id/key, export multiple files, or use existing templates. ## Installation diff --git a/cmd/ripoff-export/ripoff_export.go b/cmd/ripoff-export/ripoff_export.go index 3da8e45..806dc09 100644 --- a/cmd/ripoff-export/ripoff_export.go +++ b/cmd/ripoff-export/ripoff_export.go @@ -23,7 +23,9 @@ func errAttr(err error) slog.Attr { func main() { // Define flags var excludeTables stringSliceFlag + var ignoreOnUpdateColumns stringSliceFlag flag.Var(&excludeTables, "exclude", "Exclude specific tables from export (can be specified multiple times)") + flag.Var(&ignoreOnUpdateColumns, "ignore-on-update", "Columns to ignore during updates but include in initial export (can be specified multiple times)") // Parse flags flag.Parse() @@ -56,6 +58,33 @@ func main() { os.Exit(1) } + // Load existing data if ignore-on-update is specified and directory exists + var existingData *ripoff.RipoffFile + if len(ignoreOnUpdateColumns) > 0 && err == nil && !os.IsNotExist(err) { + // Directory exists and we have ignore-on-update columns, try to load existing data + slog.Info("Loading existing YAML data to preserve ignore-on-update column values") + // We need a temporary transaction to load enums + tempTx, tempErr := conn.Begin(ctx) + if tempErr != nil { + slog.Error("Could not create temporary transaction for loading existing data", errAttr(tempErr)) + os.Exit(1) + } + enums, tempErr := ripoff.GetEnumValues(ctx, tempTx) + _ = tempTx.Rollback(ctx) // Clean up temp transaction, ignore error since it's temporary + if tempErr != nil { + slog.Error("Could not load enums for existing data", errAttr(tempErr)) + os.Exit(1) + } + existingRipoff, tempErr := ripoff.RipoffFromDirectory(exportDirectory, enums) + if tempErr != nil { + slog.Warn("Could not load existing YAML data, will use fresh database values", errAttr(tempErr)) + existingData = nil + } else { + existingData = &existingRipoff + slog.Info(fmt.Sprintf("Loaded existing data with %d rows", len(existingData.Rows))) + } + } + // Directory exists, delete it after verifying that it's safe to do so. if err == nil && !os.IsNotExist(err) { err = filepath.WalkDir(exportDirectory, func(path string, entry os.DirEntry, err error) error { @@ -97,8 +126,9 @@ func main() { } }() - // Pass the excluded tables to the export function - ripoffFile, err := ripoff.ExportToRipoff(ctx, tx, excludeTables) + // Pass the excluded tables and ignore-on-update columns to the export function + // Use ExportToRipoffWithExisting to preserve ignore-on-update column values + ripoffFile, err := ripoff.ExportToRipoffWithExisting(ctx, tx, excludeTables, ignoreOnUpdateColumns, existingData) if err != nil { slog.Error("Could not assemble ripoff file from database", errAttr(err)) os.Exit(1) diff --git a/db.go b/db.go index 54e47b2..66730b4 100644 --- a/db.go +++ b/db.go @@ -164,6 +164,23 @@ func buildQueryForRow(primaryKeys PrimaryKeysResult, rowId string, row Row, depe columns := []string{} values := []string{} setStatements := []string{} + + // Extract ignore-on-update columns if present + ignoreOnUpdateColumns := map[string]bool{} + if ignoreOnUpdateRaw, hasIgnoreOnUpdate := row["~ignore_on_update"]; hasIgnoreOnUpdate { + switch v := ignoreOnUpdateRaw.(type) { + // Coming from yaml + case []interface{}: + for _, curr := range v { + ignoreOnUpdateColumns[curr.(string)] = true + } + // Coming from Go, probably a test + case []string: + for _, curr := range v { + ignoreOnUpdateColumns[curr] = true + } + } + } onConflictColumn := "" if hasPrimaryKeysForTable { @@ -187,6 +204,10 @@ func buildQueryForRow(primaryKeys PrimaryKeysResult, rowId string, row Row, depe if column == "~conflict" { continue } + // Ignore-on-update metadata, not a database column. + if column == "~ignore_on_update" { + continue + } // Explicit dependencies, for foreign keys to non-primary keys. if column == "~dependencies" { dependencies := []string{} @@ -212,7 +233,10 @@ func buildQueryForRow(primaryKeys PrimaryKeysResult, rowId string, row Row, depe if valueRaw == nil { columns = append(columns, pq.QuoteIdentifier(column)) values = append(values, "NULL") - setStatements = append(setStatements, fmt.Sprintf("%s = %s", pq.QuoteIdentifier(column), "NULL")) + // Only add to setStatements if this column is not ignored on update + if !ignoreOnUpdateColumns[column] { + setStatements = append(setStatements, fmt.Sprintf("%s = %s", pq.QuoteIdentifier(column), "NULL")) + } } else { value := fmt.Sprint(valueRaw) @@ -234,7 +258,10 @@ func buildQueryForRow(primaryKeys PrimaryKeysResult, rowId string, row Row, depe onConflictColumn = pq.QuoteIdentifier(column) } values = append(values, pq.QuoteLiteral(valuePrepared)) - setStatements = append(setStatements, fmt.Sprintf("%s = %s", pq.QuoteIdentifier(column), pq.QuoteLiteral(valuePrepared))) + // Only add to setStatements if this column is not ignored on update + if !ignoreOnUpdateColumns[column] { + setStatements = append(setStatements, fmt.Sprintf("%s = %s", pq.QuoteIdentifier(column), pq.QuoteLiteral(valuePrepared))) + } } } diff --git a/export.go b/export.go index 8933263..4edbb69 100644 --- a/export.go +++ b/export.go @@ -17,7 +17,16 @@ type RowMissingDependency struct { // Exports all rows in the database to a ripoff file. // excludeTables is a list of table names to exclude from the export. -func ExportToRipoff(ctx context.Context, tx pgx.Tx, excludeTables []string) (RipoffFile, error) { +// ignoreOnUpdateColumns is a list of column names to mark as ignored during updates. +func ExportToRipoff(ctx context.Context, tx pgx.Tx, excludeTables []string, ignoreOnUpdateColumns []string) (RipoffFile, error) { + return ExportToRipoffWithExisting(ctx, tx, excludeTables, ignoreOnUpdateColumns, nil) +} + +// ExportToRipoffWithExisting exports all rows in the database to a ripoff file, preserving existing values for ignore-on-update columns. +// excludeTables is a list of table names to exclude from the export. +// ignoreOnUpdateColumns is a list of column names to mark as ignored during updates. +// existingData is optional existing YAML data to preserve ignore-on-update column values from. +func ExportToRipoffWithExisting(ctx context.Context, tx pgx.Tx, excludeTables []string, ignoreOnUpdateColumns []string, existingData *RipoffFile) (RipoffFile, error) { ripoffFile := RipoffFile{ Rows: map[string]Row{}, } @@ -90,7 +99,17 @@ func ExportToRipoff(ctx context.Context, tx pgx.Tx, excludeTables []string) (Rip // A map of fieldName -> tableName to convert values to literal:(...) literalFields := map[string]string{} ids := []string{} + // Track columns that should be ignored on update + var ignoreOnUpdateFields []string for i, field := range fields { + // Check if this column should be ignored on update + for _, ignoreCol := range ignoreOnUpdateColumns { + if field.Name == ignoreCol { + ignoreOnUpdateFields = append(ignoreOnUpdateFields, field.Name) + break + } + } + // Null columns are still exported since we don't know if there is a default or not (at least not at time of writing). if columns[i] == nil { ripoffRow[field.Name] = nil @@ -172,6 +191,23 @@ func ExportToRipoff(ctx context.Context, tx pgx.Tx, excludeTables []string) (Rip for fieldName, toTable := range literalFields { ripoffRow[fieldName] = fmt.Sprintf("%s:literal(%s)", toTable, ripoffRow[fieldName]) } + // Add ignore-on-update metadata if any columns should be ignored + if len(ignoreOnUpdateFields) > 0 { + ripoffRow["~ignore_on_update"] = ignoreOnUpdateFields + } + + // Preserve existing values for ignore-on-update columns if existing data is provided + if existingData != nil { + if existingRow, exists := existingData.Rows[rowKey]; exists { + for _, ignoreCol := range ignoreOnUpdateColumns { + if existingValue, hasExistingValue := existingRow[ignoreCol]; hasExistingValue { + // Preserve the existing value instead of using the current database value + ripoffRow[ignoreCol] = existingValue + } + } + } + } + ripoffFile.Rows[rowKey] = ripoffRow } } diff --git a/export_test.go b/export_test.go index 6d72d77..6b71446 100644 --- a/export_test.go +++ b/export_test.go @@ -8,6 +8,7 @@ import ( "runtime" "strings" "testing" + "time" "github.com/jackc/pgx/v5" "github.com/lib/pq" @@ -23,7 +24,7 @@ func runExportTestData(t *testing.T, ctx context.Context, tx pgx.Tx, testDir str require.NoError(t, err) // Generate new ripoff file. - ripoffFile, err := ExportToRipoff(ctx, tx, []string{}) + ripoffFile, err := ExportToRipoff(ctx, tx, []string{}, []string{}) require.NoError(t, err) // Ensure ripoff file matches expected output. @@ -140,7 +141,7 @@ func TestExcludeFlag(t *testing.T) { // Test 1: Exclude a single table t.Run("Single exclude", func(t *testing.T) { - ripoffFile, err := ExportToRipoff(ctx, tx, []string{"exclude_me"}) + ripoffFile, err := ExportToRipoff(ctx, tx, []string{"exclude_me"}, []string{}) require.NoError(t, err) // Verify that ripoffFile.Rows contains rows from include_me but not exclude_me @@ -199,7 +200,7 @@ func TestExcludeFlag(t *testing.T) { // Test 2: Exclude multiple tables t.Run("Multiple excludes", func(t *testing.T) { - ripoffFile, err := ExportToRipoff(ctx, tx, []string{"exclude_me", "also_exclude_me"}) + ripoffFile, err := ExportToRipoff(ctx, tx, []string{"exclude_me", "also_exclude_me"}, []string{}) require.NoError(t, err) // Verify that ripoffFile.Rows contains rows from include_me but not from the excluded tables @@ -256,3 +257,177 @@ func TestExcludeFlag(t *testing.T) { require.Equal(t, 0, alsoExcludeCount, "Expected 0 rows from also_exclude_me table") }) } + +// TestIgnoreOnUpdateFlag tests that the ignore-on-update flag properly marks columns +func TestIgnoreOnUpdateFlag(t *testing.T) { + envUrl := os.Getenv("RIPOFF_TEST_DATABASE_URL") + if envUrl == "" { + envUrl = "postgres:///ripoff-test-db" + } + ctx := context.Background() + conn, err := pgx.Connect(ctx, envUrl) + if err != nil { + require.NoError(t, err) + } + defer conn.Close(ctx) + + // Start a transaction that we'll roll back at the end + tx, err := conn.Begin(ctx) + require.NoError(t, err) + defer func() { + err := tx.Rollback(ctx) + require.NoError(t, err) + }() + + // Create a table with columns including created_at and updated_at + _, err = tx.Exec(ctx, ` + CREATE TABLE test_table ( + id SERIAL PRIMARY KEY, + name TEXT, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW(), + description TEXT + ); + + INSERT INTO test_table (name, description) VALUES + ('test 1', 'first test'), + ('test 2', 'second test'); + `) + require.NoError(t, err) + + // Test 1: Ignore created_at on update + t.Run("Ignore created_at on update", func(t *testing.T) { + ripoffFile, err := ExportToRipoff(ctx, tx, []string{}, []string{"created_at"}) + require.NoError(t, err) + + // Verify that ripoffFile.Rows contains rows with created_at but marked for ignore on update + rowCount := 0 + for rowId, row := range ripoffFile.Rows { + if strings.HasPrefix(rowId, "test_table:") { + rowCount++ + // Should have all columns including created_at + _, hasName := row["name"] + _, hasDescription := row["description"] + _, hasCreatedAt := row["created_at"] + _, hasUpdatedAt := row["updated_at"] + + require.True(t, hasName, "Expected name column to be present") + require.True(t, hasDescription, "Expected description column to be present") + require.True(t, hasCreatedAt, "Expected created_at column to be present") + require.True(t, hasUpdatedAt, "Expected updated_at column to be present") + + // Should have ignore-on-update metadata + ignoreOnUpdate, hasIgnoreOnUpdate := row["~ignore_on_update"] + require.True(t, hasIgnoreOnUpdate, "Expected ~ignore_on_update metadata to be present") + + ignoreList, ok := ignoreOnUpdate.([]string) + require.True(t, ok, "Expected ~ignore_on_update to be a slice of strings") + require.Equal(t, []string{"created_at"}, ignoreList, "Expected created_at to be in ignore list") + } + } + require.Equal(t, 2, rowCount, "Expected 2 rows from test_table") + }) + + // Test 2: Ignore multiple columns on update + t.Run("Ignore multiple columns on update", func(t *testing.T) { + ripoffFile, err := ExportToRipoff(ctx, tx, []string{}, []string{"created_at", "updated_at"}) + require.NoError(t, err) + + // Verify that ripoffFile.Rows contains rows with both timestamp columns marked for ignore + rowCount := 0 + for rowId, row := range ripoffFile.Rows { + if strings.HasPrefix(rowId, "test_table:") { + rowCount++ + // Should have all columns including both timestamp columns + _, hasName := row["name"] + _, hasDescription := row["description"] + _, hasCreatedAt := row["created_at"] + _, hasUpdatedAt := row["updated_at"] + + require.True(t, hasName, "Expected name column to be present") + require.True(t, hasDescription, "Expected description column to be present") + require.True(t, hasCreatedAt, "Expected created_at column to be present") + require.True(t, hasUpdatedAt, "Expected updated_at column to be present") + + // Should have ignore-on-update metadata with both columns + ignoreOnUpdate, hasIgnoreOnUpdate := row["~ignore_on_update"] + require.True(t, hasIgnoreOnUpdate, "Expected ~ignore_on_update metadata to be present") + + ignoreList, ok := ignoreOnUpdate.([]string) + require.True(t, ok, "Expected ~ignore_on_update to be a slice of strings") + require.ElementsMatch(t, []string{"created_at", "updated_at"}, ignoreList, "Expected both timestamp columns to be in ignore list") + } + } + require.Equal(t, 2, rowCount, "Expected 2 rows from test_table") + }) + + // Test 3: No ignore-on-update metadata when no columns specified + t.Run("No ignore metadata when no columns specified", func(t *testing.T) { + ripoffFile, err := ExportToRipoff(ctx, tx, []string{}, []string{}) + require.NoError(t, err) + + // Verify that ripoffFile.Rows contains rows without ignore-on-update metadata + rowCount := 0 + for rowId, row := range ripoffFile.Rows { + if strings.HasPrefix(rowId, "test_table:") { + rowCount++ + // Should NOT have ignore-on-update metadata + _, hasIgnoreOnUpdate := row["~ignore_on_update"] + require.False(t, hasIgnoreOnUpdate, "Expected no ~ignore_on_update metadata when no columns specified") + } + } + require.Equal(t, 2, rowCount, "Expected 2 rows from test_table") + }) + + // Test 4: Verify ignore-on-update preserves existing values during re-export + t.Run("Preserve existing values on re-export", func(t *testing.T) { + // First export with ignore-on-update flags + ripoffFile1, err := ExportToRipoff(ctx, tx, []string{}, []string{"created_at", "updated_at"}) + require.NoError(t, err) + + // Get the original timestamp values from first export + var originalCreatedAt, originalUpdatedAt string + for rowId, row := range ripoffFile1.Rows { + if strings.HasPrefix(rowId, "test_table:") { + originalCreatedAt = row["created_at"].(string) + originalUpdatedAt = row["updated_at"].(string) + break + } + } + + // Simulate time passing and database changes + // Add a small delay to ensure different timestamps + time.Sleep(10 * time.Millisecond) + // Update the database with new timestamp values + _, err = tx.Exec(ctx, "UPDATE test_table SET updated_at = NOW() + INTERVAL '1 second' WHERE id = 1") + require.NoError(t, err) + + // Second export with same ignore-on-update flags + // This should preserve the original timestamp values, not use the new database values + ripoffFile2, err := ExportToRipoffWithExisting(ctx, tx, []string{}, []string{"created_at", "updated_at"}, &ripoffFile1) + require.NoError(t, err) + + // Check that the timestamp values are preserved from first export + for rowId, row := range ripoffFile2.Rows { + if strings.HasPrefix(rowId, "test_table:") { + currentCreatedAt := row["created_at"].(string) + currentUpdatedAt := row["updated_at"].(string) + + t.Logf("Original created_at: %s, Current created_at: %s", originalCreatedAt, currentCreatedAt) + t.Logf("Original updated_at: %s, Current updated_at: %s", originalUpdatedAt, currentUpdatedAt) + + // These assertions should pass with the new logic + require.Equal(t, originalCreatedAt, currentCreatedAt, "created_at should be preserved from original export") + require.Equal(t, originalUpdatedAt, currentUpdatedAt, "updated_at should be preserved from original export") + + // Verify the metadata is still present + ignoreOnUpdate, hasIgnoreOnUpdate := row["~ignore_on_update"] + require.True(t, hasIgnoreOnUpdate, "Expected ~ignore_on_update metadata to be present") + ignoreList, ok := ignoreOnUpdate.([]string) + require.True(t, ok, "Expected ~ignore_on_update to be a slice of strings") + require.ElementsMatch(t, []string{"created_at", "updated_at"}, ignoreList, "Expected both timestamps in ignore list") + break + } + } + }) +}