From 55e47be1f766b5cb705ff86ac0cb3e7ed768b207 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 22:21:01 -0600 Subject: [PATCH 01/31] add clean task --- config.nims | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/config.nims b/config.nims index 189a8fb..76ba5bf 100644 --- a/config.nims +++ b/config.nims @@ -3,19 +3,24 @@ switch("nimcache", ".nimcache") task buildimporter, "Build ormin_importer": exec "nim c -o:./tools/ormin_importer tools/ormin_importer" -task test, "Run all test suite": - buildimporterTask() +task clean, "Clean generated files": rmFile("tests/forum_model_sqlite.nim") rmFile("tests/model_sqlite.nim") + rmFile("tests/forum_model_postgres.nim") + rmFile("tests/model_postgre.nim") + +task test, "Run all test suite": + buildimporterTask() + cleanTask() + exec "nim c -f -r tests/tfeature" exec "nim c -f -r tests/tcommon" exec "nim c -f -r tests/tsqlite" task test_postgres, "Run PostgreSQL test suite": # Skip PostgreSQL tests as they require a running PostgreSQL server - rmFile("tests/forum_model_postgres.nim") - rmFile("tests/model_postgre.nim") + cleanTask() exec "nim c -r -d:postgre tests/tfeature" exec "nim c -r -d:postgre tests/tcommon" From 072f949d47671fb241e47632e6cf3703889e5f03 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 22:31:37 -0600 Subject: [PATCH 02/31] add composite key test --- tests/model_sqlite.sql | 16 ++++++++++++- tests/tcommon.nim | 52 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/tests/model_sqlite.sql b/tests/model_sqlite.sql index 2c8a41d..a966917 100644 --- a/tests/model_sqlite.sql +++ b/tests/model_sqlite.sql @@ -22,4 +22,18 @@ create table if not exists tb_timestamp( create table if not exists tb_json( typjson json not null -); \ No newline at end of file +); + +create table if not exists tb_composite_pk( + pk1 integer not null, + pk2 integer not null, + message text not null, + primary key (pk1, pk2) +); + +create table if not exists tb_composite_fk( + id integer not null, + fk1 integer not null, + fk2 integer not null, + foreign key (fk1, fk2) references tb_composite_pk(pk1, pk2) +); diff --git a/tests/tcommon.nim b/tests/tcommon.nim index 6df6db6..b6c30d9 100644 --- a/tests/tcommon.nim +++ b/tests/tcommon.nim @@ -338,3 +338,55 @@ suite "json": select tb_json(typjson) produce json check res == %*js.mapIt(%*{"typjson": it}) + + +let cps = [ + (1, 1, "one-one"), + (1, 2, "one-two"), + (2, 1, "two-one") +] + +suite "composite_pk_insert": + setup: + db.dropTable(sqlFile, "tb_composite_pk") + db.createTable(sqlFile, "tb_composite_pk") + + test "insert": + for r in cps: + query: + insert tb_composite_pk(pk1 = ?r[0], pk2 = ?r[1], message = ?r[2]) + check db.getValue(sql"select count(*) from tb_composite_pk") == $cps.len + + test "json": + for r in cps: + let v = %*{"pk1": r[0], "pk2": r[1], "message": r[2]} + query: + insert tb_composite_pk(pk1 = %v["pk1"], pk2 = %v["pk2"], message = %v["message"]) + check db.getValue(sql"select count(*) from tb_composite_pk") == $cps.len + + +suite "composite_pk": + db.dropTable(sqlFile, "tb_composite_pk") + db.createTable(sqlFile, "tb_composite_pk") + + let insertSql = sql"insert into tb_composite_pk(pk1, pk2, message) values (?, ?, ?)" + for r in cps: + db.exec(insertSql, r[0], r[1], r[2]) + doAssert db.getValue(sql"select count(*) from tb_composite_pk") == $cps.len + + test "query": + let res = query: + select tb_composite_pk(pk1, pk2, message) + check res == cps + + test "where": + let res = query: + select tb_composite_pk(pk1, pk2, message) + where pk1 == ?cps[0][0] + check res == cps.filterIt(it[0] == cps[0][0]) + + test "json": + let res = query: + select tb_composite_pk(pk1, pk2, message) + produce json + check res == %*cps.mapIt(%*{"pk1": it[0], "pk2": it[1], "message": it[2]}) From 520ec8217e10f49b4d5f76a5a46f238e6cb6d4aa Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 22:51:29 -0600 Subject: [PATCH 03/31] add postgres setup scripts --- config.nims | 9 ++++++-- tests/model_postgre.sql | 23 ++++++++++++++++----- tools/ormin_importer.nim | 39 ++++++++++++++++++++++++++++------- tools/setup_postgres.sh | 33 +++++++++++++++++++++++++++++ tools/setup_postgres.sql | 21 +++++++++++++++++++ tools/setup_postgres_role.sql | 9 ++++++++ 6 files changed, 119 insertions(+), 15 deletions(-) create mode 100644 tools/setup_postgres.sh create mode 100644 tools/setup_postgres.sql create mode 100644 tools/setup_postgres_role.sql diff --git a/config.nims b/config.nims index 76ba5bf..70ed511 100644 --- a/config.nims +++ b/config.nims @@ -18,9 +18,14 @@ task test, "Run all test suite": exec "nim c -f -r tests/tcommon" exec "nim c -f -r tests/tsqlite" +task setup_postgres, "Ensure local Postgres has test DB/user": + # Use a simple script to avoid Nim/psql quoting pitfalls + exec "bash -lc 'bash tools/setup_postgres.sh'" + task test_postgres, "Run PostgreSQL test suite": - # Skip PostgreSQL tests as they require a running PostgreSQL server cleanTask() + buildimporterTask() + setup_postgresTask() exec "nim c -r -d:postgre tests/tfeature" exec "nim c -r -d:postgre tests/tcommon" @@ -32,4 +37,4 @@ task buildexamples, "Build examples: chat and forum": selfExec "js examples/chat/frontend" selfExec "c examples/forum/forum" selfExec "c examples/forum/forumproto" - selfExec "c examples/tweeter/src/tweeter" \ No newline at end of file + selfExec "c examples/tweeter/src/tweeter" diff --git a/tests/model_postgre.sql b/tests/model_postgre.sql index 52aabe5..03ab382 100644 --- a/tests/model_postgre.sql +++ b/tests/model_postgre.sql @@ -12,15 +12,28 @@ create table if not exists tb_float( ); create table if not exists tb_string( - typstring varchar not null + typstring text not null ); create table if not exists tb_timestamp( - dt timestamp not null, - dtn timestamptz not null, - dtz timestamptz not null + dt1 timestamp not null, + dt2 timestamp not null ); create table if not exists tb_json( typjson json not null -); \ No newline at end of file +); + +create table if not exists tb_composite_pk( + pk1 integer not null, + pk2 integer not null, + message text not null, + primary key (pk1, pk2) +); + +create table if not exists tb_composite_fk( + id integer not null, + fk1 integer not null, + fk2 integer not null, + foreign key (fk1, fk2) references tb_composite_pk(pk1, pk2) +); diff --git a/tools/ormin_importer.nim b/tools/ormin_importer.nim index 38c402c..53ca518 100644 --- a/tools/ormin_importer.nim +++ b/tools/ormin_importer.nim @@ -125,17 +125,40 @@ proc collectTables(n: SqlNode; t: var KnownTables) = typ: typ, primaryKey: hasAttribute(it, {nkPrimaryKey}), refs: hasRefs(it)) + # Handle table-level foreign keys, including composite FKs for i in 1../dev/null 2>&1; then + psql -v ON_ERROR_STOP=1 -U postgres -d "$db" -f "$file" "$@" + fi +} + +run_psql_cmd() { + local db="$1"; shift + local cmd="$1"; shift + if ! psql -v ON_ERROR_STOP=1 -d "$db" -c "$cmd" "$@" >/dev/null 2>&1; then + psql -v ON_ERROR_STOP=1 -U postgres -d "$db" -c "$cmd" "$@" + fi +} + +# Create role 'test' if needed +run_psql_file postgres tools/setup_postgres_role.sql + +# Create database 'test' if needed (must be outside DO block) +if ! psql -v ON_ERROR_STOP=1 -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = 'test'" | grep -q 1; then + run_psql_cmd postgres "CREATE DATABASE test OWNER test" +fi + +# Grant privileges on public schema +run_psql_cmd test "GRANT ALL PRIVILEGES ON SCHEMA public TO test" + +echo "Postgres test DB/user ensured (role 'test', db 'test')." + diff --git a/tools/setup_postgres.sql b/tools/setup_postgres.sql new file mode 100644 index 0000000..b0dc3c8 --- /dev/null +++ b/tools/setup_postgres.sql @@ -0,0 +1,21 @@ +DO +$$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'test') THEN + CREATE ROLE test LOGIN PASSWORD 'test'; + END IF; +END +$$; + +DO +$$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_database WHERE datname = 'test') THEN + CREATE DATABASE test OWNER test; + END IF; +END +$$; + +\connect test +GRANT ALL PRIVILEGES ON SCHEMA public TO test; + diff --git a/tools/setup_postgres_role.sql b/tools/setup_postgres_role.sql new file mode 100644 index 0000000..6a19451 --- /dev/null +++ b/tools/setup_postgres_role.sql @@ -0,0 +1,9 @@ +DO +$$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'test') THEN + CREATE ROLE test LOGIN PASSWORD 'test'; + END IF; +END +$$; + From eff70fe55e46129c4754e0694eaf5a4ff2217589 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 22:54:00 -0600 Subject: [PATCH 04/31] add postgres setup scripts --- tests/tcommon.nim | 2 +- tests/tfeature.nim | 2 +- tests/tpostgre.nim | 2 +- tools/setup_postgres.sh | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/tcommon.nim b/tests/tcommon.nim index b6c30d9..fd94a84 100644 --- a/tests/tcommon.nim +++ b/tests/tcommon.nim @@ -8,7 +8,7 @@ when defined(postgre): const backend = DbBackend.postgre importModel(backend, "model_postgre") const sqlFileName = "model_postgre.sql" - let db {.global.} = open("localhost", "test", "test", "test") + let db {.global.} = open("localhost", "test", "test", "test_ormin") else: from db_connector/db_sqlite import exec, getValue diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 7849d85..36e5b39 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -13,7 +13,7 @@ when defined postgre: const backend = DbBackend.postgre importModel(backend, "forum_model_postgres") const sqlFileName = "forum_model_postgres.sql" - let db {.global.} = open("localhost", "test", "test", "test") + let db {.global.} = open("localhost", "test", "test", "test_ormin") else: from db_connector/db_sqlite import exec, getValue diff --git a/tests/tpostgre.nim b/tests/tpostgre.nim index ff7db69..bb19e96 100644 --- a/tests/tpostgre.nim +++ b/tests/tpostgre.nim @@ -8,7 +8,7 @@ import ./utils importModel(DbBackend.postgre, "model_postgre") let - db {.global.} = open("localhost", "test", "test", "test") + db {.global.} = open("localhost", "test", "test", "test_ormin") testDir = currentSourcePath.parentDir() sqlFile = testDir / "model_postgre.sql" diff --git a/tools/setup_postgres.sh b/tools/setup_postgres.sh index 9f6e3d4..ff7fff3 100644 --- a/tools/setup_postgres.sh +++ b/tools/setup_postgres.sh @@ -22,12 +22,12 @@ run_psql_cmd() { run_psql_file postgres tools/setup_postgres_role.sql # Create database 'test' if needed (must be outside DO block) -if ! psql -v ON_ERROR_STOP=1 -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = 'test'" | grep -q 1; then - run_psql_cmd postgres "CREATE DATABASE test OWNER test" +if ! psql -v ON_ERROR_STOP=1 -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = 'test_ormin'" | grep -q 1; then + run_psql_cmd postgres "CREATE DATABASE test_ormin OWNER test" fi # Grant privileges on public schema -run_psql_cmd test "GRANT ALL PRIVILEGES ON SCHEMA public TO test" +run_psql_cmd test_ormin "GRANT ALL PRIVILEGES ON SCHEMA public TO test" -echo "Postgres test DB/user ensured (role 'test', db 'test')." +echo "Postgres test DB/user ensured (role 'test', db 'test_ormin')." From 131c079d37fafa57d994342b3ecbe355cb248d76 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 23:09:20 -0600 Subject: [PATCH 05/31] fix postgres on macosx --- config.nims | 9 ++++++--- ormin/ormin_postgre.nim | 2 +- tests/tcommon.nim | 3 +++ tests/tfeature.nim | 3 +++ tests/tpostgre.nim | 4 ++++ 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/config.nims b/config.nims index 70ed511..7014d21 100644 --- a/config.nims +++ b/config.nims @@ -26,10 +26,13 @@ task test_postgres, "Run PostgreSQL test suite": cleanTask() buildimporterTask() setup_postgresTask() + # Pre-generate Postgres models to avoid include timing issues + exec "./tools/ormin_importer tests/forum_model_postgres.sql" + exec "./tools/ormin_importer tests/model_postgre.sql" - exec "nim c -r -d:postgre tests/tfeature" - exec "nim c -r -d:postgre tests/tcommon" - exec "nim c -r tests/tpostgre" + exec "nim c -f -d:nimDebugDlOpen -r -d:postgre tests/tfeature" + exec "nim c -f -d:nimDebugDlOpen -r -d:postgre tests/tcommon" + exec "nim c -f -d:nimDebugDlOpen -r tests/tpostgre" task buildexamples, "Build examples: chat and forum": buildimporterTask() diff --git a/ormin/ormin_postgre.nim b/ormin/ormin_postgre.nim index 7f5248c..527eb4e 100644 --- a/ormin/ormin_postgre.nim +++ b/ormin/ormin_postgre.nim @@ -219,7 +219,7 @@ template startQuery*(db: DbConn; s: PStmt) = template stopQuery*(db: DbConn; s: PStmt) = pqclear(queryResult) -template stepQuery*(db: DbConn; s: PStmt; returnsData: int): bool = +template stepQuery*(db: DbConn; s: PStmt; returnsData: bool): bool = inc queryI queryI < queryLen diff --git a/tests/tcommon.nim b/tests/tcommon.nim index fd94a84..9d361d8 100644 --- a/tests/tcommon.nim +++ b/tests/tcommon.nim @@ -3,6 +3,9 @@ import ormin import ormin/db_utils when defined(postgre): + when defined(macosx): + {.passL: " " & gorge("pkg-config --libs libpq").} + {.passL: "-Wl,-rpath,/opt/homebrew/lib/postgresql@14".} from db_postgres import exec, getValue const backend = DbBackend.postgre diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 36e5b39..ee32101 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -8,6 +8,9 @@ when NimVersion < "1.2.0": import ./compat let testDir = currentSourcePath.parentDir() when defined postgre: + when defined(macosx): + {.passL: " " & gorge("pkg-config --libs libpq").} + {.passL: "-Wl,-rpath,/opt/homebrew/lib/postgresql@14".} from db_connector/db_postgres import exec, getValue const backend = DbBackend.postgre diff --git a/tests/tpostgre.nim b/tests/tpostgre.nim index bb19e96..ac890d0 100644 --- a/tests/tpostgre.nim +++ b/tests/tpostgre.nim @@ -2,6 +2,10 @@ import unittest, json, strutils, macros, times, os, sequtils import db_connector/postgres import ormin +when defined(macosx): + {.passL: " " & gorge("pkg-config --libs libpq").} + {.passL: "-Wl,-rpath,/opt/homebrew/lib/postgresql@14".} + from db_connector/postgres import exec, getValue import ./utils From 2d991594d485db44511cacf6d4f3716d821902aa Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 23:39:58 -0600 Subject: [PATCH 06/31] fix postgres --- ormin/db_utils.nim | 6 +++--- ormin/ormin_postgre.nim | 2 +- ormin/queries.nim | 21 +++++++++++++-------- tests/forum_model_postgres.sql | 12 ++++++------ tests/model_postgre.sql | 5 +++-- tests/tcommon.nim | 5 ++--- tests/tfeature.nim | 1 - tests/tpostgre.nim | 22 +++++++++++----------- 8 files changed, 39 insertions(+), 35 deletions(-) diff --git a/ormin/db_utils.nim b/ormin/db_utils.nim index f424657..5446bad 100644 --- a/ormin/db_utils.nim +++ b/ormin/db_utils.nim @@ -1,6 +1,6 @@ import db_connector/db_common, strutils, strformat, re -from db_connector/db_postgres import nil -from db_connector/db_sqlite import nil +import db_connector/db_postgres as db_postgres +import db_connector/db_sqlite as db_sqlite type DbConn = db_postgres.DbConn | db_sqlite.DbConn @@ -35,4 +35,4 @@ proc dropTable*(db: DbConn; sqlFile, name: string) = when defined(postgre): db.exec(sql("drop table if exists " & n & " cascade")) else: - db.exec(sql("drop table if exists " & n)) \ No newline at end of file + db.exec(sql("drop table if exists " & n)) diff --git a/ormin/ormin_postgre.nim b/ormin/ormin_postgre.nim index 527eb4e..fce1ab9 100644 --- a/ormin/ormin_postgre.nim +++ b/ormin/ormin_postgre.nim @@ -212,7 +212,7 @@ template startQuery*(db: DbConn; s: PStmt) = nil, nil, nil, 0) if pqResultStatus(queryResult) == PGRES_COMMAND_OK: discard # insert does not returns data in pg - elif pqResultStatus(queryResult) != PGRES_TUPLES_OK: dbError(db) + elif pqResultStatus(queryResult) != PGRES_TUPLES_OK: ormin_postgre.dbError(db) var queryI {.inject.} = cint(-1) var queryLen {.inject.} = pqntuples(queryResult) diff --git a/ormin/queries.nim b/ormin/queries.nim index 12c7c7b..6a3577e 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -143,11 +143,12 @@ proc checkBool(a: DbType; n: NimNode) = macros.error "expected type 'bool', but got: " & $a, n proc checkInt(a: DbType; n: NimNode) = - if a.kind != dbInt: + if a.kind notin {dbInt, dbSerial}: macros.error "expected type 'int', but got: " & $a, n proc checkCompatible(a, b: DbType; n: NimNode) = - if a.kind != b.kind: + # Treat serial and int as compatible + if not (a.kind == b.kind or (a.kind == dbSerial and b.kind == dbInt) or (a.kind == dbInt and b.kind == dbSerial)): macros.error "incompatible types: " & $a & " and " & $b, n proc checkCompatibleSet(a, b: DbType; n: NimNode) = @@ -454,6 +455,8 @@ proc generateRoutine(name: NimNode, q: QueryBuilder; let prepare = newVarStmt(prepStmt, newCall(bindSym"prepareStmt", ident"db", newLit(sql))) let body = newStmtList() + # Ensure the prepared statement is created before binding/starting the query + body.add prepare var finalParams = newNimNode(nnkFormalParams) if q.retTypeIsJson: @@ -471,7 +474,12 @@ proc generateRoutine(name: NimNode, q: QueryBuilder; if k != nnkIteratorDef: rtyp = nnkBracketExpr.newTree(ident"seq", rtyp) finalParams.add rtyp - finalParams.add newIdentDefs(ident"db", ident("DbConn")) + when dbBackend == DbBackend.postgre: + finalParams.add newIdentDefs(ident"db", newTree(nnkDotExpr, ident"ormin_postgre", ident"DbConn")) + elif dbBackend == DbBackend.sqlite: + finalParams.add newIdentDefs(ident"db", newTree(nnkDotExpr, ident"ormin_sqlite", ident"DbConn")) + else: + finalParams.add newIdentDefs(ident"db", ident("DbConn")) var i = 1 if q.params.len > 0: body.add newCall(bindSym"startBindings", prepStmt, newLit(q.params.len)) @@ -869,14 +877,11 @@ proc queryImpl(q: QueryBuilder; body: NimNode; attempt, produceJson: bool): NimN if b.kind == nnkCommand: queryh(b, q) else: macros.error "illformed query", b let sql = queryAsString(q, body) - let prepStmt = genSym(nskVar) + let prepStmt = genSym(nskLet) let res = genSym(nskVar) - let prepStmtCall = newCall(bindSym"prepareStmt", ident"db", newLit sql) result = newTree( if q.retType.len > 0: nnkStmtListExpr else: nnkStmtList, - # really hack-ish - newGlobalVar(prepStmt, newCall(bindSym"typeof", prepStmtCall), newEmptyNode()), - getAst(once(newAssignment(prepStmt, prepStmtCall))) + newLetStmt(prepStmt, newCall(bindSym"prepareStmt", ident"db", newLit sql)) ) let rtyp = if q.retType.len > 1 or q.retType.len == 0: q.retType diff --git a/tests/forum_model_postgres.sql b/tests/forum_model_postgres.sql index 8ed5457..8ce68b0 100644 --- a/tests/forum_model_postgres.sql +++ b/tests/forum_model_postgres.sql @@ -1,5 +1,5 @@ create table if not exists thread( - id integer primary key, + id serial primary key, name varchar(100) not null, views integer not null, modified timestamp not null default CURRENT_TIMESTAMP @@ -8,7 +8,7 @@ create table if not exists thread( create unique index if not exists ThreadNameIx on thread (name); create table if not exists person( - id integer primary key, + id serial primary key, name varchar(20) not null, password varchar(32) not null, email varchar(30) not null, @@ -22,7 +22,7 @@ create table if not exists person( create unique index if not exists UserNameIx on person (name); create table if not exists post( - id integer primary key, + id serial primary key, author integer not null, ip varchar(20) not null, header varchar(100) not null, @@ -35,7 +35,7 @@ create table if not exists post( ); create table if not exists session( - id integer primary key, + id serial primary key, ip varchar(20) not null, password varchar(32) not null, userid integer not null, @@ -44,7 +44,7 @@ create table if not exists session( ); create table if not exists antibot( - id integer primary key, + id serial primary key, ip varchar(20) not null, answer varchar(30) not null, created timestamp not null default CURRENT_TIMESTAMP @@ -57,4 +57,4 @@ create table if not exists error( ); create index PersonStatusIdx on person(status); -create index PostByAuthorIdx on post(thread, author); \ No newline at end of file +create index PostByAuthorIdx on post(thread, author); diff --git a/tests/model_postgre.sql b/tests/model_postgre.sql index 03ab382..ac6c4e9 100644 --- a/tests/model_postgre.sql +++ b/tests/model_postgre.sql @@ -16,8 +16,9 @@ create table if not exists tb_string( ); create table if not exists tb_timestamp( - dt1 timestamp not null, - dt2 timestamp not null + dt timestamp not null, + dtn timestamptz not null, + dtz timestamptz not null ); create table if not exists tb_json( diff --git a/tests/tcommon.nim b/tests/tcommon.nim index 9d361d8..e407516 100644 --- a/tests/tcommon.nim +++ b/tests/tcommon.nim @@ -4,14 +4,13 @@ import ormin/db_utils when defined(postgre): when defined(macosx): - {.passL: " " & gorge("pkg-config --libs libpq").} {.passL: "-Wl,-rpath,/opt/homebrew/lib/postgresql@14".} - from db_postgres import exec, getValue + import db_connector/db_postgres as db_postgres const backend = DbBackend.postgre importModel(backend, "model_postgre") const sqlFileName = "model_postgre.sql" - let db {.global.} = open("localhost", "test", "test", "test_ormin") + let db {.global.} = db_postgres.open("localhost", "test", "test", "test_ormin") else: from db_connector/db_sqlite import exec, getValue diff --git a/tests/tfeature.nim b/tests/tfeature.nim index ee32101..798d3f3 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -9,7 +9,6 @@ let testDir = currentSourcePath.parentDir() when defined postgre: when defined(macosx): - {.passL: " " & gorge("pkg-config --libs libpq").} {.passL: "-Wl,-rpath,/opt/homebrew/lib/postgresql@14".} from db_connector/db_postgres import exec, getValue diff --git a/tests/tpostgre.nim b/tests/tpostgre.nim index ac890d0..c9b63cf 100644 --- a/tests/tpostgre.nim +++ b/tests/tpostgre.nim @@ -1,18 +1,18 @@ import unittest, json, strutils, macros, times, os, sequtils -import db_connector/postgres +# Postgres connection handled through ormin_postgre backend import ormin +import ormin/ormin_postgre as ormin_postgre +import db_connector/db_postgres as db_postgres +import ormin/db_utils when defined(macosx): - {.passL: " " & gorge("pkg-config --libs libpq").} {.passL: "-Wl,-rpath,/opt/homebrew/lib/postgresql@14".} -from db_connector/postgres import exec, getValue -import ./utils importModel(DbBackend.postgre, "model_postgre") let - db {.global.} = open("localhost", "test", "test", "test_ormin") + db {.global.} = ormin_postgre.open("localhost", "test", "test", "test_ormin") testDir = currentSourcePath.parentDir() sqlFile = testDir / "model_postgre.sql" @@ -59,23 +59,23 @@ suite "timestamp_insert": test "insert": query: insert tb_timestamp(dt = ?dt1, dtn = ?dtn1, dtz = ?dtz1) - check db.getValue(sql"select count(*) from tb_timestamp") == "1" + check db_postgres.getValue(db, sql"select count(*) from tb_timestamp") == "1" test "json": query: insert tb_timestamp(dt = %dtjson1["dt"], dtn = %dtjson1["dtn"], dtz = %dtjson1["dtz"]) - check db.getValue(sql"select count(*) from tb_timestamp") == "1" + check db_postgres.getValue(db, sql"select count(*) from tb_timestamp") == "1" suite "timestamp": db.dropTable(sqlFile, "tb_timestamp") db.createTable(sqlFile, "tb_timestamp") - db.exec(insertSql, dtStr1, dtnStr1, dtzStr1) - db.exec(insertSql, dtStr2, dtnStr2, dtzStr2) - db.exec(insertSql, dtStr3, dtnStr3, dtzStr3) - doAssert db.getValue(sql"select count(*) from tb_timestamp") == "3" + db_postgres.exec(db, insertSql, dtStr1, dtnStr1, dtzStr1) + db_postgres.exec(db, insertSql, dtStr2, dtnStr2, dtzStr2) + db_postgres.exec(db, insertSql, dtStr3, dtnStr3, dtzStr3) + doAssert db_postgres.getValue(db, sql"select count(*) from tb_timestamp") == "3" test "query": let res = query: From d5707d097cf83bf8ab0c59bdb310a967df89ce44 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 23:55:46 -0600 Subject: [PATCH 07/31] fix postgres --- tests/tfeature.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 798d3f3..2641073 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -529,7 +529,7 @@ suite "query": test "insert_return_answer": # test returning non-id parameter - let expectedanswer = "just insert" + let expectedanswer = "just insert another" let answer = query: insert antibot(id = 9, ip = "", answer = ?expectedanswer) returning answer @@ -538,7 +538,7 @@ suite "query": test "insert_return_id_auto": # test returning id column let answer = query: - insert antibot(ip = "", answer = "just another insert") + insert antibot(ip = "", answer = "just auto insert") returning id check answer == 10 From 62669fb767e543b9bb7a532aacaa846c97a3960a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:17:46 -0600 Subject: [PATCH 08/31] fix postgres test, document using db_connector --- README.md | 8 ++++++++ tests/tfeature.nim | 3 +++ 2 files changed, 11 insertions(+) diff --git a/README.md b/README.md index cac5a52..01cd9e0 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,14 @@ for row in db.postsIter(userId): Both forms accept parameters matching the `?`/`%` placeholders and produce the same return types as an inline `query` block. +## Running Arbitrary SQL + +The standard `db_connector` APIs can be imported and used. For example: + +```nim +discard db.getValue(sql"select setval('antibot_id_seq', 10, false)") +``` + ## Additional Facilities - **Protocol DSL** – The `protocol` macro lets you describe paired server/client handlers that communicate via JSON messages. Sections use keywords like `recv`, `broadcast` and `send`, and every server block must be mirrored by a client block. The chat example demonstrates this code generation. diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 2641073..31eaf8a 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -537,6 +537,9 @@ suite "query": test "insert_return_id_auto": # test returning id column + when defined(postgre): + # fix postgres sequence so next nextval returns 10 + discard db.getValue(sql"select setval('antibot_id_seq', ?, ?)", 10, false) let answer = query: insert antibot(ip = "", answer = "just auto insert") returning id From 764035ec02e8c5074f1799e0d92d1a441ceee5ba Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:18:10 -0600 Subject: [PATCH 09/31] fix postgres test, document using db_connector --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ec23b13..7d21850 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: postgresql version: '14' postgresql user: 'test' postgresql password: 'test' - postgresql db: 'test' + postgresql db: 'test_ormin' - name: Install system dependencies run: | From 41beb00c0312a6f40ce9a59204f5a3b94620c748 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:19:33 -0600 Subject: [PATCH 10/31] run pg tests --- .github/workflows/test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7d21850..e7d0662 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,8 +52,11 @@ jobs: nimble --useSystemNim install karax -y nimble --useSystemNim install jester -y - - name: Run test + - name: Run test sqlite run: nimble --useSystemNim test + + - name: Run test postgres + run: nimble --useSystemNim test_postgres env: PGPASSWORD: test From 5f69404672011bff7c58395af22d65956c80d93d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:28:06 -0600 Subject: [PATCH 11/31] run pg tests --- .github/workflows/test.yml | 1 + ormin.nimble | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e7d0662..96f3e83 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,6 +51,7 @@ jobs: nimble --useSystemNim install -d nimble --useSystemNim install karax -y nimble --useSystemNim install jester -y + nimble --useSystemNim install websocket -y - name: Run test sqlite run: nimble --useSystemNim test diff --git a/ormin.nimble b/ormin.nimble index 23e4d37..895cac6 100644 --- a/ormin.nimble +++ b/ormin.nimble @@ -12,11 +12,9 @@ installExt = @["nim"] requires "nim >= 2.0.0" requires "db_connector >= 0.1.0" -feature "websocket": +feature "examples": requires "websocket >= 0.2.2" -feature "karax": requires "karax" -feature "jester": requires "jester" include "config.nims" From 1db14481ceb7b40da253a789b7283fbe960a0247 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:34:00 -0600 Subject: [PATCH 12/31] run pg tests --- .github/workflows/test.yml | 12 +++++------- config.nims | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 96f3e83..5909efe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,13 +33,11 @@ jobs: with: nim-version: ${{ matrix.nim }} - - name: Setup postgresql - uses: harmon758/postgresql-action@v1 - with: - postgresql version: '14' - postgresql user: 'test' - postgresql password: 'test' - postgresql db: 'test_ormin' + - name: Start PostgreSQL + run: | + sudo systemctl start postgresql@14-main + sudo -u postgres psql -c "CREATE USER test WITH SUPERUSER CREATEDB CREATEROLE PASSWORD 'test';" + sudo -u postgres psql -c "CREATE DATABASE test_ormin OWNER test;" - name: Install system dependencies run: | diff --git a/config.nims b/config.nims index 7014d21..a567662 100644 --- a/config.nims +++ b/config.nims @@ -25,7 +25,7 @@ task setup_postgres, "Ensure local Postgres has test DB/user": task test_postgres, "Run PostgreSQL test suite": cleanTask() buildimporterTask() - setup_postgresTask() + # setup_postgresTask() # Pre-generate Postgres models to avoid include timing issues exec "./tools/ormin_importer tests/forum_model_postgres.sql" exec "./tools/ormin_importer tests/model_postgre.sql" From 446b1b6a8c2fbc5e15888ed426404bce6364f25e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:38:29 -0600 Subject: [PATCH 13/31] run pg tests --- .github/workflows/test.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5909efe..67b33c6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,6 +5,22 @@ on: [push] jobs: build: runs-on: ubuntu-latest + + services: + postgres: + image: postgres:14 + env: + POSTGRES_PASSWORD: test + POSTGRES_USER: test + POSTGRES_DB: test_ormin + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + strategy: matrix: nim: From 30bf3e4a405e22ea6f4cf25e12fcc84981c81368 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:39:17 -0600 Subject: [PATCH 14/31] run pg tests --- .github/workflows/test.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67b33c6..1ce6367 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,12 +49,6 @@ jobs: with: nim-version: ${{ matrix.nim }} - - name: Start PostgreSQL - run: | - sudo systemctl start postgresql@14-main - sudo -u postgres psql -c "CREATE USER test WITH SUPERUSER CREATEDB CREATEROLE PASSWORD 'test';" - sudo -u postgres psql -c "CREATE DATABASE test_ormin OWNER test;" - - name: Install system dependencies run: | sudo apt-get update From 55b8724006a7762a42d5e64bf131d1954f5cd939 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:44:29 -0600 Subject: [PATCH 15/31] run pg tests --- .github/workflows/test.yml | 5 ----- ormin/db_utils.nim | 11 +++++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1ce6367..644034a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,11 +49,6 @@ jobs: with: nim-version: ${{ matrix.nim }} - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libpcre3-dev - - name: Install dependencies run: | nimble --useSystemNim install -d diff --git a/ormin/db_utils.nim b/ormin/db_utils.nim index 5446bad..a56c2ec 100644 --- a/ormin/db_utils.nim +++ b/ormin/db_utils.nim @@ -1,4 +1,4 @@ -import db_connector/db_common, strutils, strformat, re +import db_connector/db_common, strutils, strformat, pegs import db_connector/db_postgres as db_postgres import db_connector/db_sqlite as db_sqlite @@ -6,10 +6,13 @@ type DbConn = db_postgres.DbConn | db_sqlite.DbConn iterator tablePairs(sqlFile: string): tuple[name, model: string] = let f = readFile(sqlFile) + let pat = peg"start <- \s* 'create' \s+ 'table' \s+ ('if' \s+ 'not' \s+ 'exists' \s+)? { [A-Za-z_][A-Za-z0-9_]* }" for m in f.split(';'): - if m.strip() != "" and - m =~ re"\n*create\s+table(\s+if\s+not\s+exists)?\s+(\w+)": - yield (matches[1], m) + let s = m.toLowerAscii() + if m.strip() != "": + var caps = newSeq[string](1) + if s.match(pat, caps): + yield (caps[0], m) proc createTable*(db: DbConn; sqlFile: string) = for _, m in tablePairs(sqlFile): From 3bb38d806439eaf67f9dd07d05c6aba3651caf09 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 01:06:55 -0600 Subject: [PATCH 16/31] use pegs --- .gitignore | 1 + ormin.nimble | 2 +- ormin/db_utils.nim | 15 +++++++++++-- tests/db_utils_case_quoted.sql | 12 +++++++++++ tests/tdb_utils.nim | 39 ++++++++++++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 tests/db_utils_case_quoted.sql create mode 100644 tests/tdb_utils.nim diff --git a/.gitignore b/.gitignore index d5f9ada..28353e0 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ examples/tweeter/src/tweeterdeps/ deps/ nim.cfg .nimcache +tests/tdb_utils diff --git a/ormin.nimble b/ormin.nimble index 895cac6..9e582f6 100644 --- a/ormin.nimble +++ b/ormin.nimble @@ -1,6 +1,6 @@ # Package -version = "0.3.0" +version = "0.4.0" author = "Araq" description = "Prepared SQL statement generator. A lightweight ORM." license = "MIT" diff --git a/ormin/db_utils.nim b/ormin/db_utils.nim index a56c2ec..1e32efe 100644 --- a/ormin/db_utils.nim +++ b/ormin/db_utils.nim @@ -6,12 +6,23 @@ type DbConn = db_postgres.DbConn | db_sqlite.DbConn iterator tablePairs(sqlFile: string): tuple[name, model: string] = let f = readFile(sqlFile) - let pat = peg"start <- \s* 'create' \s+ 'table' \s+ ('if' \s+ 'not' \s+ 'exists' \s+)? { [A-Za-z_][A-Za-z0-9_]* }" + let pat = peg""" + start <- \s* 'create' \s+ 'table' \s+ ('if' \s+ 'not' \s+ 'exists' \s+)? ident + ident <- quoted / unquoted + quoted <- '"' { (!'"' .)+ } '"' + unquoted <- { [A-Za-z_][A-Za-z0-9_]* } + """ for m in f.split(';'): let s = m.toLowerAscii() if m.strip() != "": + var cleaned = newSeq[string]() + for ln in s.splitLines(): + let t = ln.strip() + if not t.startsWith("--"): + cleaned.add ln + let sc = cleaned.join("\n") var caps = newSeq[string](1) - if s.match(pat, caps): + if sc.match(pat, caps): yield (caps[0], m) proc createTable*(db: DbConn; sqlFile: string) = diff --git a/tests/db_utils_case_quoted.sql b/tests/db_utils_case_quoted.sql new file mode 100644 index 0000000..ef8db4e --- /dev/null +++ b/tests/db_utils_case_quoted.sql @@ -0,0 +1,12 @@ +-- lower case, upper case, and quoted table names + create table lower_table ( + id integer primary key + ); + + CREATE TABLE UPPER_TABLE ( + id integer primary key + ); + + create table "Quoted_Table" ( + id integer primary key + ); diff --git a/tests/tdb_utils.nim b/tests/tdb_utils.nim new file mode 100644 index 0000000..3268b63 --- /dev/null +++ b/tests/tdb_utils.nim @@ -0,0 +1,39 @@ +import unittest, os +import db_connector/db_common +from db_connector/db_sqlite import open, exec, getValue +import ormin/db_utils + +let + db {.global.} = open(":memory:", "", "", "") + testDir = currentSourcePath.parentDir() + sqlFile = testDir / "db_utils_case_quoted.sql" + +let sqlContent = """ +-- lower case, upper case, and quoted table names + create table lower_table ( + id integer primary key + ); + + CREATE TABLE UPPER_TABLE ( + id integer primary key + ); + + create table "Quoted_Table" ( + id integer primary key + ); +""" + +writeFile(sqlFile, sqlContent) + +suite "db_utils: case and quoted names": + test "createTable creates all tables from SQL file": + db.createTable(sqlFile) + let countAll = db.getValue(sql"select count(*) from sqlite_master where type='table' and name in ('lower_table','UPPER_TABLE','Quoted_Table')") + check countAll == "3" + + test "createTable with specific lowercased name matches quoted": + # Use a new in-memory DB for isolation + let db2 = open(":memory:", "", "", "") + db2.createTable(sqlFile, "quoted_table") + let countQuoted = db2.getValue(sql"select count(*) from sqlite_master where type='table' and name = 'Quoted_Table'") + check countQuoted == "1" From 6587455289f2071a52ab98761b218879af9eac4d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 01:08:56 -0600 Subject: [PATCH 17/31] use pegs --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 644034a..46ee496 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,6 @@ name: CI -on: [push] +on: [push, pull_request] jobs: build: @@ -65,4 +65,4 @@ jobs: PGPASSWORD: test - name: Build examples - run: nimble buildexamples \ No newline at end of file + run: nimble buildexamples From 697341591dad0b28ec4835550d884481e7465393 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sat, 13 Sep 2025 06:20:52 -0600 Subject: [PATCH 18/31] Use parsesql for table name detection --- ormin/db_utils.nim | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/ormin/db_utils.nim b/ormin/db_utils.nim index 1e32efe..401852e 100644 --- a/ormin/db_utils.nim +++ b/ormin/db_utils.nim @@ -1,4 +1,5 @@ -import db_connector/db_common, strutils, strformat, pegs +import db_connector/db_common, strutils, strformat +import std/parsesql import db_connector/db_postgres as db_postgres import db_connector/db_sqlite as db_sqlite @@ -6,24 +7,18 @@ type DbConn = db_postgres.DbConn | db_sqlite.DbConn iterator tablePairs(sqlFile: string): tuple[name, model: string] = let f = readFile(sqlFile) - let pat = peg""" - start <- \s* 'create' \s+ 'table' \s+ ('if' \s+ 'not' \s+ 'exists' \s+)? ident - ident <- quoted / unquoted - quoted <- '"' { (!'"' .)+ } '"' - unquoted <- { [A-Za-z_][A-Za-z0-9_]* } - """ for m in f.split(';'): - let s = m.toLowerAscii() - if m.strip() != "": - var cleaned = newSeq[string]() - for ln in s.splitLines(): - let t = ln.strip() - if not t.startsWith("--"): - cleaned.add ln - let sc = cleaned.join("\n") - var caps = newSeq[string](1) - if sc.match(pat, caps): - yield (caps[0], m) + let stmt = m.strip() + if stmt.len == 0: continue + try: + let ast = parseSql(stmt) + if ast.len > 0: + let node = ast[0] + if node.kind in {nkCreateTable, nkCreateTableIfNotExists}: + let tableName = node[0].strVal.toLowerAscii() + yield (tableName, stmt) + except SqlParseError: + discard proc createTable*(db: DbConn; sqlFile: string) = for _, m in tablePairs(sqlFile): From 9beb13d510acda145b0671c64aac2031b10f2281 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 07:04:05 -0600 Subject: [PATCH 19/31] updates --- ormin/db_utils.nim | 29 ++++++++++++++++------------- tests/tdb_utils.nim | 11 ++++++++++- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/ormin/db_utils.nim b/ormin/db_utils.nim index 401852e..e5f7a5c 100644 --- a/ormin/db_utils.nim +++ b/ormin/db_utils.nim @@ -5,20 +5,23 @@ import db_connector/db_sqlite as db_sqlite type DbConn = db_postgres.DbConn | db_sqlite.DbConn -iterator tablePairs(sqlFile: string): tuple[name, model: string] = +iterator tablePairs*(sqlFile: string): tuple[name, model: string] = + # Parse the entire SQL file and iterate statements via the SQL parser let f = readFile(sqlFile) - for m in f.split(';'): - let stmt = m.strip() - if stmt.len == 0: continue - try: - let ast = parseSql(stmt) - if ast.len > 0: - let node = ast[0] - if node.kind in {nkCreateTable, nkCreateTableIfNotExists}: - let tableName = node[0].strVal.toLowerAscii() - yield (tableName, stmt) - except SqlParseError: - discard + let ast = parseSql(f) + if ast.len > 0: + # ast is a statement list; iterate each statement node + for i in 0 ..< ast.len: + let node = ast[i] + if node.kind in {nkCreateTable, nkCreateTableIfNotExists}: + let tableName = node[0].strVal.toLowerAscii() + yield (tableName, $node) + else: + # Fallback: ast might be a single statement (not a list) + let node = ast + if node.kind in {nkCreateTable, nkCreateTableIfNotExists}: + let tableName = node[0].strVal.toLowerAscii() + yield (tableName, $node) proc createTable*(db: DbConn; sqlFile: string) = for _, m in tablePairs(sqlFile): diff --git a/tests/tdb_utils.nim b/tests/tdb_utils.nim index 3268b63..78241b9 100644 --- a/tests/tdb_utils.nim +++ b/tests/tdb_utils.nim @@ -1,4 +1,4 @@ -import unittest, os +import unittest, os, sequtils import db_connector/db_common from db_connector/db_sqlite import open, exec, getValue import ormin/db_utils @@ -26,6 +26,15 @@ let sqlContent = """ writeFile(sqlFile, sqlContent) suite "db_utils: case and quoted names": + test "check tables names": + let pairs = tablePairs(sqlFile).toSeq() + check pairs.len == 3 + check pairs[0][0] == ("lower_table") + check pairs[1][0] == ("upper_table") + check pairs[2][0] == ("quoted_table") + + check pairs[0][1] == ("create table lower_table (id integer primary key)") + test "createTable creates all tables from SQL file": db.createTable(sqlFile) let countAll = db.getValue(sql"select count(*) from sqlite_master where type='table' and name in ('lower_table','UPPER_TABLE','Quoted_Table')") From 32e6594b1cadf03ef152d7c4a0bda75d4e9360e8 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 07:05:22 -0600 Subject: [PATCH 20/31] updates --- tests/tdb_utils.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/tdb_utils.nim b/tests/tdb_utils.nim index 78241b9..1c01c86 100644 --- a/tests/tdb_utils.nim +++ b/tests/tdb_utils.nim @@ -33,7 +33,9 @@ suite "db_utils: case and quoted names": check pairs[1][0] == ("upper_table") check pairs[2][0] == ("quoted_table") - check pairs[0][1] == ("create table lower_table (id integer primary key)") + + echo pairs[0][1].repr() + check pairs[0][1] == ("create table lower_table (id integer primary key);") test "createTable creates all tables from SQL file": db.createTable(sqlFile) From 547d731f7e6074ef37ad568f2b99be4086873cc6 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 07:08:51 -0600 Subject: [PATCH 21/31] updates --- tests/db_utils_case_quoted.sql | 2 +- tests/tdb_utils.nim | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/db_utils_case_quoted.sql b/tests/db_utils_case_quoted.sql index ef8db4e..d3c0121 100644 --- a/tests/db_utils_case_quoted.sql +++ b/tests/db_utils_case_quoted.sql @@ -7,6 +7,6 @@ id integer primary key ); - create table "Quoted_Table" ( + create table "Quoted Table" ( id integer primary key ); diff --git a/tests/tdb_utils.nim b/tests/tdb_utils.nim index 1c01c86..c3b6653 100644 --- a/tests/tdb_utils.nim +++ b/tests/tdb_utils.nim @@ -18,7 +18,7 @@ let sqlContent = """ id integer primary key ); - create table "Quoted_Table" ( + create table "Quoted Table" ( id integer primary key ); """ @@ -31,20 +31,21 @@ suite "db_utils: case and quoted names": check pairs.len == 3 check pairs[0][0] == ("lower_table") check pairs[1][0] == ("upper_table") - check pairs[2][0] == ("quoted_table") + check pairs[2][0] == ("quoted table") - echo pairs[0][1].repr() - check pairs[0][1] == ("create table lower_table (id integer primary key);") + check pairs[0][1] == "create table lower_table(id integer primary key );" + check pairs[1][1] == "create table UPPER_TABLE(id integer primary key );" + check pairs[2][1] == "create table \"Quoted Table\"(id integer primary key );" test "createTable creates all tables from SQL file": db.createTable(sqlFile) - let countAll = db.getValue(sql"select count(*) from sqlite_master where type='table' and name in ('lower_table','UPPER_TABLE','Quoted_Table')") + let countAll = db.getValue(sql"select count(*) from sqlite_master where type='table' and name in ('lower_table','UPPER_TABLE','Quoted Table')") check countAll == "3" test "createTable with specific lowercased name matches quoted": # Use a new in-memory DB for isolation let db2 = open(":memory:", "", "", "") - db2.createTable(sqlFile, "quoted_table") - let countQuoted = db2.getValue(sql"select count(*) from sqlite_master where type='table' and name = 'Quoted_Table'") + db2.createTable(sqlFile, "quoted table") + let countQuoted = db2.getValue(sql"select count(*) from sqlite_master where type='table' and name = 'Quoted Table'") check countQuoted == "1" From 236f72179d936a325e8dad5fd99046778f9efd57 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 07:10:21 -0600 Subject: [PATCH 22/31] updates --- tests/db_utils_case_quoted.sql | 6 ++++++ tests/tdb_utils.nim | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/db_utils_case_quoted.sql b/tests/db_utils_case_quoted.sql index d3c0121..cc25649 100644 --- a/tests/db_utils_case_quoted.sql +++ b/tests/db_utils_case_quoted.sql @@ -7,6 +7,12 @@ id integer primary key ); + -- std sql quoted table name create table "Quoted Table" ( id integer primary key ); + + -- sqlite quoted table name + create table `Quoted Table2` ( + id integer primary key + ); diff --git a/tests/tdb_utils.nim b/tests/tdb_utils.nim index c3b6653..435eaa4 100644 --- a/tests/tdb_utils.nim +++ b/tests/tdb_utils.nim @@ -18,9 +18,15 @@ let sqlContent = """ id integer primary key ); + -- std sql quoted table name create table "Quoted Table" ( id integer primary key ); + + -- sqlite quoted table name + create table `Quoted Table2` ( + id integer primary key + ); """ writeFile(sqlFile, sqlContent) @@ -28,10 +34,11 @@ writeFile(sqlFile, sqlContent) suite "db_utils: case and quoted names": test "check tables names": let pairs = tablePairs(sqlFile).toSeq() - check pairs.len == 3 + check pairs.len == 4 check pairs[0][0] == ("lower_table") check pairs[1][0] == ("upper_table") check pairs[2][0] == ("quoted table") + check pairs[3][0] == ("quoted table2") check pairs[0][1] == "create table lower_table(id integer primary key );" From 2f35e8c7a937179305165bd270386fc020134b55 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 07:29:40 -0600 Subject: [PATCH 23/31] fix path handling to allow direct sql from strings to be used --- ormin/db_utils.nim | 22 ++++++++++++++-------- tests/tdb_utils.nim | 7 +++---- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/ormin/db_utils.nim b/ormin/db_utils.nim index e5f7a5c..767fc74 100644 --- a/ormin/db_utils.nim +++ b/ormin/db_utils.nim @@ -1,14 +1,15 @@ import db_connector/db_common, strutils, strformat -import std/parsesql +import std/parsesql, std/paths import db_connector/db_postgres as db_postgres import db_connector/db_sqlite as db_sqlite +export paths + type DbConn = db_postgres.DbConn | db_sqlite.DbConn -iterator tablePairs*(sqlFile: string): tuple[name, model: string] = +iterator tablePairs*(sql: string): tuple[name, model: string] = # Parse the entire SQL file and iterate statements via the SQL parser - let f = readFile(sqlFile) - let ast = parseSql(f) + let ast = parseSql(sql) if ast.len > 0: # ast is a statement list; iterate each statement node for i in 0 ..< ast.len: @@ -23,25 +24,30 @@ iterator tablePairs*(sqlFile: string): tuple[name, model: string] = let tableName = node[0].strVal.toLowerAscii() yield (tableName, $node) -proc createTable*(db: DbConn; sqlFile: string) = +iterator tablePairs*(sql: Path): tuple[name, model: string] = + let f = readFile($sql) + for n, m in tablePairs(f): + yield (n, m) + +proc createTable*(db: DbConn; sqlFile: Path) = for _, m in tablePairs(sqlFile): db.exec(sql(m)) -proc createTable*(db: DbConn; sqlFile, name: string) = +proc createTable*(db: DbConn; sqlFile: Path, name: string) = for n, m in tablePairs(sqlFile): if n == name: db.exec(sql(m)) return raiseAssert &"table: {name} not found in: {sqlFile}" -proc dropTable*(db: DbConn; sqlFile: string) = +proc dropTable*(db: DbConn; sqlFile: Path) = for n, _ in tablePairs(sqlFile): when defined(postgre): db.exec(sql("drop table if exists " & n & " cascade")) else: db.exec(sql("drop table if exists " & n)) -proc dropTable*(db: DbConn; sqlFile, name: string) = +proc dropTable*(db: DbConn; sqlFile: Path, name: string) = for n, _ in tablePairs(sqlFile): if n == name: when defined(postgre): diff --git a/tests/tdb_utils.nim b/tests/tdb_utils.nim index 435eaa4..8e1b54c 100644 --- a/tests/tdb_utils.nim +++ b/tests/tdb_utils.nim @@ -6,7 +6,7 @@ import ormin/db_utils let db {.global.} = open(":memory:", "", "", "") testDir = currentSourcePath.parentDir() - sqlFile = testDir / "db_utils_case_quoted.sql" + sqlFile = Path(testDir / "db_utils_case_quoted.sql") let sqlContent = """ -- lower case, upper case, and quoted table names @@ -29,18 +29,17 @@ let sqlContent = """ ); """ -writeFile(sqlFile, sqlContent) +writeFile($sqlFile, sqlContent) suite "db_utils: case and quoted names": test "check tables names": - let pairs = tablePairs(sqlFile).toSeq() + let pairs = tablePairs(sqlContent).toSeq() check pairs.len == 4 check pairs[0][0] == ("lower_table") check pairs[1][0] == ("upper_table") check pairs[2][0] == ("quoted table") check pairs[3][0] == ("quoted table2") - check pairs[0][1] == "create table lower_table(id integer primary key );" check pairs[1][1] == "create table UPPER_TABLE(id integer primary key );" check pairs[2][1] == "create table \"Quoted Table\"(id integer primary key );" From 9d4c0d296c2c29037e3f10569d82fd17896c86d9 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 07:37:51 -0600 Subject: [PATCH 24/31] fix impoprts --- ormin/db_utils.nim | 12 ++++++++++-- {tools => ormin}/parsesql_tmp.nim | 0 tests/tfeature.nim | 2 +- tools/ormin_importer.nim | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) rename {tools => ormin}/parsesql_tmp.nim (100%) diff --git a/ormin/db_utils.nim b/ormin/db_utils.nim index 767fc74..54ec3fb 100644 --- a/ormin/db_utils.nim +++ b/ormin/db_utils.nim @@ -1,15 +1,23 @@ +import std/paths import db_connector/db_common, strutils, strformat -import std/parsesql, std/paths import db_connector/db_postgres as db_postgres import db_connector/db_sqlite as db_sqlite +import parsesql_tmp # import std/parsesql + export paths type DbConn = db_postgres.DbConn | db_sqlite.DbConn iterator tablePairs*(sql: string): tuple[name, model: string] = # Parse the entire SQL file and iterate statements via the SQL parser - let ast = parseSql(sql) + var ast: SqlNode + try: + ast = parseSql(sql) + except SqlParseError as e: + echo "SQL Parse Error:\n", sql + raise e + if ast.len > 0: # ast is a statement list; iterate each statement node for i in 0 ..< ast.len: diff --git a/tools/parsesql_tmp.nim b/ormin/parsesql_tmp.nim similarity index 100% rename from tools/parsesql_tmp.nim rename to ormin/parsesql_tmp.nim diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 31eaf8a..33fadd4 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -25,7 +25,7 @@ else: var memoryPath = testDir & "/" & ":memory:" let db {.global.} = open(memoryPath, "", "", "") -var sqlFilePath = testDir & "/" & sqlFileName +var sqlFilePath = Path(testDir & "/" & sqlFileName) type Person = tuple[id: int, diff --git a/tools/ormin_importer.nim b/tools/ormin_importer.nim index 53ca518..f348099 100644 --- a/tools/ormin_importer.nim +++ b/tools/ormin_importer.nim @@ -5,7 +5,7 @@ import streams, strutils, os, parseopt, tables import db_connector/db_common -import ./parsesql_tmp +import ../ormin/parsesql_tmp #import compiler / [ast, renderer] From 4f4dbfdd7fbeb5c4191ac0b7c36821ed0a17f9ed Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 07:53:51 -0600 Subject: [PATCH 25/31] fix impoprts --- tests/tsqlite.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tsqlite.nim b/tests/tsqlite.nim index 2d4cad7..5999c13 100644 --- a/tests/tsqlite.nim +++ b/tests/tsqlite.nim @@ -8,7 +8,7 @@ importModel(DbBackend.sqlite, "model_sqlite") let db {.global.} = open(":memory:", "", "", "") testDir = currentSourcePath.parentDir() - sqlFile = testDir / "model_sqlite.sql" + sqlFile = Path(testDir / "model_sqlite.sql") suite "Test special database types and functions of sqlite": From d10043556941629e09478af970c9cfdf58255167 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 07:58:42 -0600 Subject: [PATCH 26/31] add test --- config.nims | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.nims b/config.nims index a567662..08eca33 100644 --- a/config.nims +++ b/config.nims @@ -17,6 +17,7 @@ task test, "Run all test suite": exec "nim c -f -r tests/tfeature" exec "nim c -f -r tests/tcommon" exec "nim c -f -r tests/tsqlite" + exec "nim c -f -r tests/tdb_utils" task setup_postgres, "Ensure local Postgres has test DB/user": # Use a simple script to avoid Nim/psql quoting pitfalls @@ -32,7 +33,7 @@ task test_postgres, "Run PostgreSQL test suite": exec "nim c -f -d:nimDebugDlOpen -r -d:postgre tests/tfeature" exec "nim c -f -d:nimDebugDlOpen -r -d:postgre tests/tcommon" - exec "nim c -f -d:nimDebugDlOpen -r tests/tpostgre" + exec "nim c -f -d:nimDebugDlOpen -r -d:postgre tests/tpostgre" task buildexamples, "Build examples: chat and forum": buildimporterTask() From 3604685824cacc9f8069d53d263b392cd96665b3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 08:07:02 -0600 Subject: [PATCH 27/31] add test --- ormin/parsesql_tmp.nim | 15 ++++++++++++++- tests/tcommon.nim | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ormin/parsesql_tmp.nim b/ormin/parsesql_tmp.nim index 21677d4..d3ddbc8 100644 --- a/ormin/parsesql_tmp.nim +++ b/ormin/parsesql_tmp.nim @@ -1536,7 +1536,20 @@ proc ra(n: SqlNode, s: var SqlWriter) = rs(n, s) of nkForeignKey: s.addKeyw("foreign key") - rs(n, s) + # Render only the column identifiers inside parentheses, + # then append the REFERENCES clause (and options) after. + var i = 0 + s.add('(') + while i < n.len and n.sons[i].kind != nkReferences: + if i > 0: s.add(", ") + ra(n.sons[i], s) + inc i + s.add(')') + # If a REFERENCES node exists, render it after the column list + while i < n.len: + s.add(' ') + ra(n.sons[i], s) + inc i of nkNotNull: s.addKeyw("not null") of nkNull: diff --git a/tests/tcommon.nim b/tests/tcommon.nim index e407516..326f070 100644 --- a/tests/tcommon.nim +++ b/tests/tcommon.nim @@ -22,7 +22,7 @@ else: let testDir = currentSourcePath.parentDir() - sqlFile = testDir / sqlFileName + sqlFile = Path(testDir / sqlFileName) let From 49349fa6c8a11a588b3c315524b8c20221eb5505 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 08:08:57 -0600 Subject: [PATCH 28/31] add test --- tests/tfeature.nim | 3 ++- tests/tpostgre.nim | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 33fadd4..d796c3f 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -383,10 +383,11 @@ suite "query": check res.sortedByIt(it) == expectedpost.sortedByIt(it) test "subquery_nest3": - let res = query: + var res = query: select thread(id) where id in (select post(thread) where author in (select person(id) where id in {1, 2})) + res.sort() check res == postdata.filterIt(it.author in [1, 2]) .mapIt(it.thread) .sortedByIt(it) diff --git a/tests/tpostgre.nim b/tests/tpostgre.nim index c9b63cf..5e6aaa5 100644 --- a/tests/tpostgre.nim +++ b/tests/tpostgre.nim @@ -14,7 +14,7 @@ importModel(DbBackend.postgre, "model_postgre") let db {.global.} = ormin_postgre.open("localhost", "test", "test", "test_ormin") testDir = currentSourcePath.parentDir() - sqlFile = testDir / "model_postgre.sql" + sqlFile = Path(testDir / "model_postgre.sql") suite "Test special database types and functions of postgre": From 6d3d4ddc1f8e5f261b743fd59c93f08d35f6aecc Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 08:14:55 -0600 Subject: [PATCH 29/31] add test --- ormin/queries.nim | 7 +------ tests/tpostgre.nim | 3 ++- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 6a3577e..dc231f9 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -474,12 +474,7 @@ proc generateRoutine(name: NimNode, q: QueryBuilder; if k != nnkIteratorDef: rtyp = nnkBracketExpr.newTree(ident"seq", rtyp) finalParams.add rtyp - when dbBackend == DbBackend.postgre: - finalParams.add newIdentDefs(ident"db", newTree(nnkDotExpr, ident"ormin_postgre", ident"DbConn")) - elif dbBackend == DbBackend.sqlite: - finalParams.add newIdentDefs(ident"db", newTree(nnkDotExpr, ident"ormin_sqlite", ident"DbConn")) - else: - finalParams.add newIdentDefs(ident"db", ident("DbConn")) + finalParams.add newIdentDefs(ident"db", ident("DbConn")) var i = 1 if q.params.len > 0: body.add newCall(bindSym"startBindings", prepStmt, newLit(q.params.len)) diff --git a/tests/tpostgre.nim b/tests/tpostgre.nim index 5e6aaa5..1caee77 100644 --- a/tests/tpostgre.nim +++ b/tests/tpostgre.nim @@ -1,8 +1,9 @@ import unittest, json, strutils, macros, times, os, sequtils # Postgres connection handled through ormin_postgre backend +from db_connector/db_postgres import exec, getValue import ormin import ormin/ormin_postgre as ormin_postgre -import db_connector/db_postgres as db_postgres +# import db_connector/db_postgres as db_postgres import ormin/db_utils when defined(macosx): From f95b7c766721d59511fd2d5ba0e6b44de54e206c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 08:17:41 -0600 Subject: [PATCH 30/31] add test --- tests/tpostgre.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/tpostgre.nim b/tests/tpostgre.nim index 1caee77..0929252 100644 --- a/tests/tpostgre.nim +++ b/tests/tpostgre.nim @@ -60,23 +60,23 @@ suite "timestamp_insert": test "insert": query: insert tb_timestamp(dt = ?dt1, dtn = ?dtn1, dtz = ?dtz1) - check db_postgres.getValue(db, sql"select count(*) from tb_timestamp") == "1" + check db.getValue(sql"select count(*) from tb_timestamp") == "1" test "json": query: insert tb_timestamp(dt = %dtjson1["dt"], dtn = %dtjson1["dtn"], dtz = %dtjson1["dtz"]) - check db_postgres.getValue(db, sql"select count(*) from tb_timestamp") == "1" + check db.getValue(sql"select count(*) from tb_timestamp") == "1" suite "timestamp": db.dropTable(sqlFile, "tb_timestamp") db.createTable(sqlFile, "tb_timestamp") - db_postgres.exec(db, insertSql, dtStr1, dtnStr1, dtzStr1) - db_postgres.exec(db, insertSql, dtStr2, dtnStr2, dtzStr2) - db_postgres.exec(db, insertSql, dtStr3, dtnStr3, dtzStr3) - doAssert db_postgres.getValue(db, sql"select count(*) from tb_timestamp") == "3" + db.exec(insertSql, dtStr1, dtnStr1, dtzStr1) + db.exec(insertSql, dtStr2, dtnStr2, dtzStr2) + db.exec(insertSql, dtStr3, dtnStr3, dtzStr3) + doAssert db.getValue(sql"select count(*) from tb_timestamp") == "3" test "query": let res = query: From 3a8306c7148813d2144028e685039ca071ba1216 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 21 Sep 2025 01:34:46 -0600 Subject: [PATCH 31/31] Update tests/model_sqlite.sql --- tests/model_sqlite.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/model_sqlite.sql b/tests/model_sqlite.sql index b7ad09a..8b4ecee 100644 --- a/tests/model_sqlite.sql +++ b/tests/model_sqlite.sql @@ -36,7 +36,7 @@ create table if not exists tb_composite_fk( fk1 integer not null, fk2 integer not null, foreign key (fk1, fk2) references tb_composite_pk(pk1, pk2) - +); create table if not exists tb_blob( id integer primary key, typblob blob not null