From 735bfebc5c4dfb2ee41460a920f2b0560b77a0c6 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Thu, 8 Jan 2026 08:15:57 +0000 Subject: [PATCH] feat(csv-to-pg): add conflictDoNothing option for ON CONFLICT DO NOTHING Add a new conflictDoNothing option to the Parser, InsertOne, and InsertMany functions that generates ON CONFLICT DO NOTHING without specifying conflict columns. This is useful when the unique constraint uses a functional index with complex expressions that cannot be specified as simple column names. The metaschema_public.field table uses a unique constraint (databases_field_uniq_names_idx) that normalizes UUID field names by stripping suffixes like _id, _uuid, etc. This causes collisions when tables have both 'foo' (text) and 'foo_id' (uuid) columns. Using ON CONFLICT DO NOTHING allows the export to gracefully skip duplicate field inserts during deployment. Changes: - Add conflictDoNothing option to makeConflictClause in utils.ts - Add conflictDoNothing to InsertOne and InsertMany params - Add conflictDoNothing to Parser config and parse method - Update export-meta.ts to use conflictDoNothing for field table - Add tests for the new conflictDoNothing option --- packages/csv-to-pg/__tests__/csv2pg.test.ts | 72 +++++++++++++++++++++ packages/csv-to-pg/src/parser.ts | 9 ++- packages/csv-to-pg/src/utils.ts | 25 +++++-- pgpm/core/src/export/export-meta.ts | 5 ++ 4 files changed, 103 insertions(+), 8 deletions(-) diff --git a/packages/csv-to-pg/__tests__/csv2pg.test.ts b/packages/csv-to-pg/__tests__/csv2pg.test.ts index a01136074..3c0ecd6e0 100644 --- a/packages/csv-to-pg/__tests__/csv2pg.test.ts +++ b/packages/csv-to-pg/__tests__/csv2pg.test.ts @@ -15,6 +15,78 @@ const testCase = resolve(__dirname + '/../__fixtures__/test-case.csv'); it('noop', () => { expect(true).toBe(true); }); + +describe('conflictDoNothing', () => { + it('InsertOne with conflictDoNothing generates ON CONFLICT DO NOTHING AST', () => { + const config = { + schema: 'my-schema', + table: 'my-table', + fields: { + name: 'text' + } + }; + const types = parseTypes(config); + const stmt = InsertOne({ + schema: config.schema, + table: config.table, + types, + record: { name: 'test' }, + conflictDoNothing: true + }); + + // Verify the AST contains the ON CONFLICT DO NOTHING clause + expect(stmt.RawStmt.stmt.InsertStmt.onConflictClause).toEqual({ + action: 'ONCONFLICT_NOTHING' + }); + }); + + it('InsertMany with conflictDoNothing generates ON CONFLICT DO NOTHING AST', () => { + const config = { + schema: 'my-schema', + table: 'my-table', + fields: { + name: 'text' + } + }; + const types = parseTypes(config); + const stmt = InsertMany({ + schema: config.schema, + table: config.table, + types, + records: [ + { name: 'test1' }, + { name: 'test2' } + ], + conflictDoNothing: true + }); + + // Verify the AST contains the ON CONFLICT DO NOTHING clause + expect(stmt.RawStmt.stmt.InsertStmt.onConflictClause).toEqual({ + action: 'ONCONFLICT_NOTHING' + }); + }); + + it('InsertOne without conflictDoNothing has no conflict clause', () => { + const config = { + schema: 'my-schema', + table: 'my-table', + fields: { + name: 'text' + } + }; + const types = parseTypes(config); + const stmt = InsertOne({ + schema: config.schema, + table: config.table, + types, + record: { name: 'test' } + }); + + // Verify no conflict clause when conflictDoNothing is not set + expect(stmt.RawStmt.stmt.InsertStmt.onConflictClause).toBeUndefined(); + }); +}); + xdescribe('Insert Many', () => { it('Insert Many', async () => { const config = { diff --git a/packages/csv-to-pg/src/parser.ts b/packages/csv-to-pg/src/parser.ts index 9053ebad5..d6e6a6a4f 100644 --- a/packages/csv-to-pg/src/parser.ts +++ b/packages/csv-to-pg/src/parser.ts @@ -8,6 +8,7 @@ interface ParserConfig { table: string; singleStmts?: boolean; conflict?: string[]; + conflictDoNothing?: boolean; headers?: string[]; delimeter?: string; json?: boolean; @@ -30,7 +31,7 @@ export class Parser { async parse(data?: Record[]): Promise { const config = this.config; - const { schema, table, singleStmts, conflict, headers, delimeter } = config; + const { schema, table, singleStmts, conflict, conflictDoNothing, headers, delimeter } = config; const opts: CsvOptions = {}; if (headers) opts.headers = headers; @@ -67,7 +68,8 @@ export class Parser { table, types, record, - conflict + conflict, + conflictDoNothing }) ); return deparse(stmts); @@ -77,7 +79,8 @@ export class Parser { table, types, records, - conflict + conflict, + conflictDoNothing }); return deparse([stmt]); } diff --git a/packages/csv-to-pg/src/utils.ts b/packages/csv-to-pg/src/utils.ts index 0da82d11e..6bed31b18 100644 --- a/packages/csv-to-pg/src/utils.ts +++ b/packages/csv-to-pg/src/utils.ts @@ -130,7 +130,18 @@ const indexElem = (name: string): Node => ({ import type { OnConflictClause } from '@pgsql/types'; -const makeConflictClause = (conflictElems: string[] | undefined, fields: string[]): OnConflictClause | undefined => { +const makeConflictClause = ( + conflictElems: string[] | undefined, + fields: string[], + conflictDoNothing?: boolean +): OnConflictClause | undefined => { + // If conflictDoNothing is true, generate ON CONFLICT DO NOTHING without specifying columns + // This catches any unique constraint violation + if (conflictDoNothing) { + return { + action: 'ONCONFLICT_NOTHING' + }; + } if (!conflictElems || !conflictElems.length) return undefined; const setElems = fields.filter((el) => !conflictElems.includes(el)); if (setElems.length) { @@ -157,6 +168,7 @@ interface InsertOneParams { types: TypesMap; record: Record; conflict?: string[]; + conflictDoNothing?: boolean; } export const InsertOne = ({ @@ -164,7 +176,8 @@ export const InsertOne = ({ table, types, record, - conflict + conflict, + conflictDoNothing }: InsertOneParams): Node => ({ RawStmt: { stmt: { @@ -189,7 +202,7 @@ export const InsertOne = ({ limitOption: 'LIMIT_OPTION_DEFAULT' } }, - onConflictClause: makeConflictClause(conflict, Object.keys(types)), + onConflictClause: makeConflictClause(conflict, Object.keys(types), conflictDoNothing), override: 'OVERRIDING_NOT_SET' } }, @@ -203,6 +216,7 @@ interface InsertManyParams { types: TypesMap; records: Record[]; conflict?: string[]; + conflictDoNothing?: boolean; } export const InsertMany = ({ @@ -210,7 +224,8 @@ export const InsertMany = ({ table, types, records, - conflict + conflict, + conflictDoNothing }: InsertManyParams): Node => ({ RawStmt: { stmt: { @@ -233,7 +248,7 @@ export const InsertMany = ({ limitOption: 'LIMIT_OPTION_DEFAULT' } }, - onConflictClause: makeConflictClause(conflict, Object.keys(types)), + onConflictClause: makeConflictClause(conflict, Object.keys(types), conflictDoNothing), override: 'OVERRIDING_NOT_SET' } }, diff --git a/pgpm/core/src/export/export-meta.ts b/pgpm/core/src/export/export-meta.ts index 2ef82dc5d..387fdbf27 100644 --- a/pgpm/core/src/export/export-meta.ts +++ b/pgpm/core/src/export/export-meta.ts @@ -7,6 +7,7 @@ type FieldType = 'uuid' | 'uuid[]' | 'text' | 'text[]' | 'boolean' | 'image' | ' interface TableConfig { schema: string; table: string; + conflictDoNothing?: boolean; fields: Record; } @@ -57,6 +58,10 @@ const config: Record = { field: { schema: 'metaschema_public', table: 'field', + // Use ON CONFLICT DO NOTHING to handle the unique constraint (databases_field_uniq_names_idx) + // which normalizes UUID field names by stripping suffixes like _id, _uuid, etc. + // This causes collisions when tables have both 'foo' (text) and 'foo_id' (uuid) columns. + conflictDoNothing: true, fields: { id: 'uuid', database_id: 'uuid',