From f94d97b1761d9b267ee9c372240b69a473350afe Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Mon, 11 Aug 2025 17:28:57 +0100 Subject: [PATCH 1/2] feat: WIP. Proof of concept of using sqlite-wasm --- esbuild.config.mjs | 47 +++++++++++ package.json | 7 +- pnpm-lock.yaml | 77 ++++++++++++++++++ src/main.ts | 5 ++ src/modules/database/factory.ts | 4 +- src/modules/database/module.ts | 12 +-- .../database/sqlocal/databaseProvider.ts | 78 +++++++++++++++++++ src/modules/database/sqlocal/schema.ts | 7 ++ src/modules/main/init.ts | 5 ++ 9 files changed, 234 insertions(+), 8 deletions(-) create mode 100644 src/modules/database/sqlocal/databaseProvider.ts create mode 100644 src/modules/database/sqlocal/schema.ts diff --git a/esbuild.config.mjs b/esbuild.config.mjs index bfad87e..37838d3 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -16,6 +16,7 @@ if you want to view the source, please visit the github repository of this plugi const wasmPlugin = { name: 'wasm', setup(build) { + // Handle .wasm files build.onResolve({ filter: /\.wasm$/ }, args => { if (args.resolveDir === '') return; return { @@ -37,6 +38,52 @@ const wasmPlugin = { loader: 'js', }; }); + + // Handle SQLite bundler-friendly module + build.onResolve({ filter: /sqlite3-bundler-friendly\.mjs$/ }, args => { + if (args.path.startsWith('@sqlite.org/sqlite-wasm')) { + // Resolve to the actual file path in node_modules + return { + path: join(process.cwd(), 'node_modules', args.path), + namespace: 'sqlite-bundler', + }; + } + return { + path: join(args.resolveDir, args.path), + namespace: 'sqlite-bundler', + }; + }); + + build.onLoad({ filter: /sqlite3-bundler-friendly\.mjs$/, namespace: 'sqlite-bundler' }, async (args) => { + const moduleContents = readFileSync(args.path, 'utf8'); + const wasmPath = args.path.replace('sqlite3-bundler-friendly.mjs', 'sqlite3.wasm'); + const wasmContents = readFileSync(wasmPath); + const wasmBase64 = wasmContents.toString('base64'); + + // Replace the findWasmBinary function to return embedded WASM + const modifiedContents = moduleContents + // Replace the entire findWasmBinary function + .replace( + /function\s+findWasmBinary\s*\(\s*\)\s*\{[\s\S]*?return\s+new\s+URL\s*\(\s*['"`]sqlite3\.wasm['"`]\s*,\s*import\.meta\.url\s*\)\.href\s*;?\s*\}/g, + `function findWasmBinary() { + return "data:application/wasm;base64,${wasmBase64}"; + }` + ) + // Also replace any direct new URL(...) patterns as fallback + .replace( + /new\s+URL\s*\(\s*['"`]sqlite3\.wasm['"`]\s*,\s*import\.meta\.url\s*\)\.href/g, + `"data:application/wasm;base64,${wasmBase64}"` + ) + .replace( + /new\s+URL\s*\(\s*['"`]sqlite3\.wasm['"`]\s*,\s*import\.meta\.url\s*\)/g, + `"data:application/wasm;base64,${wasmBase64}"` + ); + + return { + contents: modifiedContents, + loader: 'js', + }; + }); } }; diff --git a/package.json b/package.json index 02ab3fc..eeaf7ee 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "@hypersphere/dity-graph": "^0.0.8", "@hypersphere/omnibus": "^0.1.6", "@jlongster/sql.js": "^1.6.7", + "@sqlite.org/sqlite-wasm": "3.50.3-build1", "@types/jsonpath": "^0.2.4", "@vanakat/plugin-api": "^0.2.1", "absurd-sql": "^0.0.54", @@ -70,6 +71,9 @@ "handlebars": "^4.7.8", "json5": "^2.2.3", "jsonpath": "^1.1.1", + "kysely": "^0.28.5", + "kysely-wasm": "^1.2.1", + "kysely-wasqlite-worker": "^1.2.1", "lodash": "^4.17.21", "markdown-table-ts": "^1.0.3", "mermaid": "11.4.0", @@ -78,6 +82,7 @@ "sql-parser-cst": "^0.33.1", "unidecode": "^1.1.0", "util": "^0.12.5", - "uuid": "^11.1.0" + "uuid": "^11.1.0", + "wa-sqlite": "^1.0.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 123fa03..c43c093 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,9 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +overrides: + updated-sql.js: link:../../../Library/pnpm/global/5/node_modules/updated-sql.js + importers: .: @@ -32,6 +35,9 @@ importers: '@jlongster/sql.js': specifier: ^1.6.7 version: 1.6.7 + '@sqlite.org/sqlite-wasm': + specifier: 3.50.3-build1 + version: 3.50.3-build1 '@types/jsonpath': specifier: ^0.2.4 version: 0.2.4 @@ -65,6 +71,15 @@ importers: jsonpath: specifier: ^1.1.1 version: 1.1.1 + kysely: + specifier: ^0.28.5 + version: 0.28.5 + kysely-wasm: + specifier: ^1.2.1 + version: 1.2.1(kysely@0.28.5) + kysely-wasqlite-worker: + specifier: ^1.2.1 + version: 1.2.1(@subframe7536/sqlite-wasm@0.5.6)(kysely@0.28.5) lodash: specifier: ^4.17.21 version: 4.17.21 @@ -92,6 +107,9 @@ importers: uuid: specifier: ^11.1.0 version: 11.1.0 + wa-sqlite: + specifier: ^1.0.0 + version: 1.0.0 devDependencies: '@changesets/cli': specifier: ^2.29.4 @@ -1226,6 +1244,13 @@ packages: '@sinonjs/fake-timers@10.3.0': resolution: {integrity: sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==} + '@sqlite.org/sqlite-wasm@3.50.3-build1': + resolution: {integrity: sha512-NU+I7KJ5kpMZNyZtJ9hOLlhQHJAA3fJhtkE7kf3C0SlGg4ayz6AkqxcaDcR4qOsrz1XP2+yM1yORaLCt55XDQg==} + hasBin: true + + '@subframe7536/sqlite-wasm@0.5.6': + resolution: {integrity: sha512-uvVJPXGUZNXp2vN19LcPEZTaDO/+r/IE/OazhczaSq95puPX/7xaEQAter3U3viK1WHdGnMfqltPj6QqRMK/2g==} + '@szmarczak/http-timer@4.0.6': resolution: {integrity: sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w==} engines: {node: '>=10'} @@ -3071,6 +3096,26 @@ packages: kolorist@1.8.0: resolution: {integrity: sha512-Y+60/zizpJ3HRH8DCss+q95yr6145JXZo46OTpFvDZWLfRCE4qChOyk1b26nMaNpfHHgxagk9dXT5OP0Tfe+dQ==} + kysely-generic-sqlite@1.2.1: + resolution: {integrity: sha512-/Bs3/Uktn04nQ9g/4oSphLMEtSHkS5+j5hbKjK5gMqXQfqr/v3V3FKtoN4pLTmo2W35hNdrIpQnBukGL1zZc6g==} + peerDependencies: + kysely: '>=0.26' + + kysely-wasm@1.2.1: + resolution: {integrity: sha512-VXzSiTu6gDmfZdCIq04hFWucvTepcgurIP+dlCONF+p3jFU3cHhEvVN1X+TUlMHt1/7t9Q/2Vr9JyxLOxv4WIw==} + peerDependencies: + kysely: '>=0.26' + + kysely-wasqlite-worker@1.2.1: + resolution: {integrity: sha512-9tXz4RldOXXFrfSTGQ5hLRX8hxbA8LskW04CMJ1unkcDz64JeuqueVDSo2pcNaq/9YcmRnGv3L7nVgca7Jm5Zg==} + peerDependencies: + '@subframe7536/sqlite-wasm': '>=0.5.0' + kysely: '>=0.26' + + kysely@0.28.5: + resolution: {integrity: sha512-rlB0I/c6FBDWPcQoDtkxi9zIvpmnV5xoIalfCMSMCa7nuA6VGA3F54TW9mEgX4DVf10sXAWCF5fDbamI/5ZpKA==} + engines: {node: '>=20.0.0'} + langium@3.0.0: resolution: {integrity: sha512-+Ez9EoiByeoTu/2BXmEaZ06iPNXM6thWJp02KfBO/raSMyCJ4jw7AkWWa+zBCTm0+Tw1Fj9FOxdqSskyN5nAwg==} engines: {node: '>=16.0.0'} @@ -4335,6 +4380,9 @@ packages: w3c-keyname@2.2.8: resolution: {integrity: sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==} + wa-sqlite@1.0.0: + resolution: {integrity: sha512-Kyybo5/BaJp76z7gDWGk2J6Hthl4NIPsE+swgraEjy3IY6r5zIR02wAs1OJH4XtJp1y3puj3Onp5eMGS0z7nUA==} + walker@1.0.8: resolution: {integrity: sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==} @@ -4393,6 +4441,9 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + zen-mitt@3.1.0: + resolution: {integrity: sha512-I0KUcnQb9rnx1/Cu9xtlJPj2e2r+yb3putg2Tjx9mjY2pVTnAzH4N/KwnIpnG+RQCz3VPj33NftGis4ug5s/2g==} + zwitch@2.0.4: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} @@ -5622,6 +5673,10 @@ snapshots: dependencies: '@sinonjs/commons': 3.0.1 + '@sqlite.org/sqlite-wasm@3.50.3-build1': {} + + '@subframe7536/sqlite-wasm@0.5.6': {} + '@szmarczak/http-timer@4.0.6': dependencies: defer-to-connect: 2.0.1 @@ -7862,6 +7917,24 @@ snapshots: kolorist@1.8.0: {} + kysely-generic-sqlite@1.2.1(kysely@0.28.5): + dependencies: + kysely: 0.28.5 + + kysely-wasm@1.2.1(kysely@0.28.5): + dependencies: + kysely: 0.28.5 + kysely-generic-sqlite: 1.2.1(kysely@0.28.5) + + kysely-wasqlite-worker@1.2.1(@subframe7536/sqlite-wasm@0.5.6)(kysely@0.28.5): + dependencies: + '@subframe7536/sqlite-wasm': 0.5.6 + kysely: 0.28.5 + kysely-generic-sqlite: 1.2.1(kysely@0.28.5) + zen-mitt: 3.1.0 + + kysely@0.28.5: {} + langium@3.0.0: dependencies: chevrotain: 11.0.3 @@ -9102,6 +9175,8 @@ snapshots: w3c-keyname@2.2.8: {} + wa-sqlite@1.0.0: {} + walker@1.0.8: dependencies: makeerror: 1.0.12 @@ -9165,4 +9240,6 @@ snapshots: yocto-queue@0.1.0: {} + zen-mitt@3.1.0: {} + zwitch@2.0.4: {} diff --git a/src/main.ts b/src/main.ts index 2b6009c..fb28446 100644 --- a/src/main.ts +++ b/src/main.ts @@ -6,6 +6,7 @@ import { provideGlobalGridOptions, } from "ag-grid-community"; import { mainModule } from "./modules/main/module"; +import { sql } from "kysely"; // Register all community features ModuleRegistry.registerModules([AllCommunityModule]); @@ -28,6 +29,10 @@ export default class SqlSealPlugin extends Plugin { const init = await this.container.get("init"); init(); + + const db = await (await this.container.get('db.provider')).get() + const data = await db.selectNoFrom(sql`sqlite_version()`).execute() + console.log(data) } onunload() {} diff --git a/src/modules/database/factory.ts b/src/modules/database/factory.ts index 63350de..e77d87b 100644 --- a/src/modules/database/factory.ts +++ b/src/modules/database/factory.ts @@ -1,10 +1,10 @@ import { App } from "obsidian"; import { SqlSealDatabase } from "./database"; import { makeInjector } from "@hypersphere/dity"; -import { DbModel } from "./module"; +import { DatabaseModule } from "./module"; -@(makeInjector()([ +@(makeInjector()([ 'app' ])) export class DatabaseFactory { diff --git a/src/modules/database/module.ts b/src/modules/database/module.ts index 8b483ec..c23420b 100644 --- a/src/modules/database/module.ts +++ b/src/modules/database/module.ts @@ -1,13 +1,15 @@ -import { asFactory, buildContainer } from "@hypersphere/dity"; -import { App } from "obsidian"; +import { asClass, asFactory, buildContainer } from "@hypersphere/dity"; +import { App, Vault } from "obsidian"; import { DatabaseFactory } from "./factory"; +import { DatabaseProvider } from "./sqlocal/databaseProvider"; export const db = buildContainer(c => c .register({ - db: asFactory(DatabaseFactory) + db: asFactory(DatabaseFactory), + provider: asClass(DatabaseProvider) }) .externals<{ app: App }>() - .exports('db') + .exports('db', 'provider') ) -export type DbModel = typeof db +export type DatabaseModule = typeof db diff --git a/src/modules/database/sqlocal/databaseProvider.ts b/src/modules/database/sqlocal/databaseProvider.ts new file mode 100644 index 0000000..be007cb --- /dev/null +++ b/src/modules/database/sqlocal/databaseProvider.ts @@ -0,0 +1,78 @@ +import { Kysely } from "kysely"; +import { Database } from "./schema"; +import { OfficialWasmDialect } from "kysely-wasm"; +import sqlite3InitModule from "@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3-bundler-friendly.mjs"; +import { App, Vault } from "obsidian"; +import { makeInjector } from "@hypersphere/dity"; +import { DatabaseModule } from "../module"; +import { sanitise } from "../../../utils/sanitiseColumn"; + +@(makeInjector()(["app"])) +export class DatabaseProvider { + constructor(private app: App) {} + private databases: Map> = new Map(); + + private _sqlite3: any = null; + private async sqlite3() { + if (this._sqlite3) { + return this._sqlite3; + } + this._sqlite3 = (await sqlite3InitModule()).oo1; + return this._sqlite3; + } + + get prefix() { + const filename = + sanitise(this.app.vault.getName()) + "___" + (this.app as any).appId; + return filename; + } + + async get( + filename: F, + ): Promise : Kysely> { + const key = filename ?? "GLOBAL"; + + const db = this.databases.get(key); + if (db) { + return db as any; // FIXME: fix typing here + } + + const path = this.prefix + "___" + key; + + try { + // Create dialect using official SQLite WASM + const dialect = new OfficialWasmDialect({ + database: async () => { + const sqlite3 = await this.sqlite3(); + if (!sqlite3) { + throw new Error("Failed to load SQLite WASM"); + } + const dbPath = path; + + return new sqlite3.DB(dbPath, "ct"); + }, + }); + + // Create Kysely instance + const db = new Kysely({ + dialect, + }); + + this.databases.set(key, db); + return db as any; + } catch (error) { + console.error("Error creating SQLite WASM database:", error); + throw error; + } + } + + async getGlobal() { + return this.get(null) + } + + async close() { + return Promise.all( + Array.from(this.databases.entries()).map(([_, db]) => db.destroy()), + ); + } +} diff --git a/src/modules/database/sqlocal/schema.ts b/src/modules/database/sqlocal/schema.ts new file mode 100644 index 0000000..d471a9e --- /dev/null +++ b/src/modules/database/sqlocal/schema.ts @@ -0,0 +1,7 @@ +export type Files = { + path: string +} + +export type Database = { + files: Files +} \ No newline at end of file diff --git a/src/modules/main/init.ts b/src/modules/main/init.ts index 4aceb24..df7ad22 100644 --- a/src/modules/main/init.ts +++ b/src/modules/main/init.ts @@ -1,5 +1,6 @@ import { makeInjector } from "@hypersphere/dity"; import { MainModule } from "./module"; +import { Plugin } from "obsidian"; type InitFn = () => void @@ -28,6 +29,7 @@ export class Init { explorerInit: InitFn ) { return () => { + settingsInit() editorInit() highlighInit() @@ -37,6 +39,9 @@ export class Init { apiInit() globalTablesInit() explorerInit() + + console.log('🚀 SQL Seal initialized with wa-sqlite test command available'); + console.log('📋 Use Ctrl/Cmd+P -> "Test wa-sqlite Implementation" to test wa-sqlite'); } } } \ No newline at end of file From cd46a77cc7c9c63886399b46f6dcdd8cf786aa08 Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Tue, 12 Aug 2025 10:28:22 +0100 Subject: [PATCH 2/2] chore: few experiments with kysely --- .../database/sqlocal/databaseProvider.ts | 49 +++- .../database/sqlocal/kyselyDatabase.ts | 227 ++++++++++++++++++ src/modules/database/sqlocal/queryDatabase.ts | 53 ++++ src/modules/database/sqlocal/schema.ts | 37 ++- tsconfig.json | 1 + 5 files changed, 358 insertions(+), 9 deletions(-) create mode 100644 src/modules/database/sqlocal/kyselyDatabase.ts create mode 100644 src/modules/database/sqlocal/queryDatabase.ts diff --git a/src/modules/database/sqlocal/databaseProvider.ts b/src/modules/database/sqlocal/databaseProvider.ts index be007cb..5a11ff0 100644 --- a/src/modules/database/sqlocal/databaseProvider.ts +++ b/src/modules/database/sqlocal/databaseProvider.ts @@ -1,16 +1,22 @@ import { Kysely } from "kysely"; -import { Database } from "./schema"; +import { DatabaseSchema } from "./schema"; import { OfficialWasmDialect } from "kysely-wasm"; import sqlite3InitModule from "@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3-bundler-friendly.mjs"; +import { Database} from '@sqlite.org/sqlite-wasm' import { App, Vault } from "obsidian"; import { makeInjector } from "@hypersphere/dity"; import { DatabaseModule } from "../module"; import { sanitise } from "../../../utils/sanitiseColumn"; +interface DbMapEntry { + database: any + kysely: Kysely +} + @(makeInjector()(["app"])) export class DatabaseProvider { constructor(private app: App) {} - private databases: Map> = new Map(); + private databases: Map = new Map(); private _sqlite3: any = null; private async sqlite3() { @@ -29,7 +35,7 @@ export class DatabaseProvider { async get( filename: F, - ): Promise : Kysely> { + ): Promise : Kysely> { const key = filename ?? "GLOBAL"; const db = this.databases.get(key); @@ -39,6 +45,8 @@ export class DatabaseProvider { const path = this.prefix + "___" + key; + let rawDatabase: Database | null = null + try { // Create dialect using official SQLite WASM const dialect = new OfficialWasmDialect({ @@ -49,16 +57,22 @@ export class DatabaseProvider { } const dbPath = path; - return new sqlite3.DB(dbPath, "ct"); + const db = new sqlite3.DB(dbPath, "ct"); + rawDatabase = db + return db }, }); + // Create Kysely instance - const db = new Kysely({ + const kysely = new Kysely({ dialect, }); - this.databases.set(key, db); + this.databases.set(key, { + kysely, + database: rawDatabase + }); return db as any; } catch (error) { console.error("Error creating SQLite WASM database:", error); @@ -67,9 +81,30 @@ export class DatabaseProvider { } async getGlobal() { - return this.get(null) + const db = await this.get(null) + const tables = await db.introspection.getTables() + if (!tables.length) { + // Generating schema + + } } + async getRawDatabaseAccess(filename: string | null = null) { + await this.get(filename) + return this.databases.get(filename ?? 'GLOBAL')!.database + } + + createGlobalTables(db: Kysely) { + db.schema + .createTable('files') + .addColumn('id', 'text') + .addColumn('name', 'text') + .addColumn('path', 'text') + .addColumn('created_at', 'datetime', c => c.defaultTo('now')) + .addColumn('modified_at', 'datetime') + .addColumn('file_size', 'numeric') + } + async close() { return Promise.all( Array.from(this.databases.entries()).map(([_, db]) => db.destroy()), diff --git a/src/modules/database/sqlocal/kyselyDatabase.ts b/src/modules/database/sqlocal/kyselyDatabase.ts new file mode 100644 index 0000000..8da6a6a --- /dev/null +++ b/src/modules/database/sqlocal/kyselyDatabase.ts @@ -0,0 +1,227 @@ +import { Kysely, sql, RawBuilder } from "kysely"; +import { uniq, uniqBy } from "lodash"; +import { ColumnDefinition } from "../../../utils/types"; +import { sanitise } from "../../../utils/sanitiseColumn"; + +const formatData = (data: Record) => { + return Object.keys(data).reduce((ret, key) => { + if (typeof data[key] === 'boolean') { + return { + ...ret, + [key]: data[key] ? 1 : 0 + } + } + if (!data[key]) { + return { + ...ret, + [key]: null + } + } + if (typeof data[key] === 'object' || Array.isArray(data[key])) { + return { + ...ret, + [key]: JSON.stringify(data[key]) + } + } + return { + ...ret, + [key]: data[key] + } + }, {}) +} + +export class KyselyDatabase { + constructor(private readonly db: Kysely) {} + + registerCustomFunction(name: string, argsCount = 1): void { + const fn = (...arg: string[]) => { + const data = { + type: name, + values: arg + } + return `SQLSEALCUSTOM(${JSON.stringify(data)})` + } + + // Note: Kysely doesn't have direct function registration like sql.js + // This would need to be handled at the dialect level or through raw SQL + console.warn(`registerCustomFunction not fully implemented for Kysely: ${name}`); + } + + async createTableText(tableName: string, fields: string[]): Promise { + const fieldsString = fields.map(f => `${f} TEXT`).join(',') + await sql`CREATE TABLE IF NOT EXISTS ${sql.table(tableName)} (${sql.raw(fieldsString)})`.execute(this.db) + } + + async recreateDatabase() { + await sql` + PRAGMA writable_schema = 1; + DELETE FROM sqlite_master; + PRAGMA writable_schema = 0; + VACUUM; + PRAGMA integrity_check; + `.execute(this.db) + } + + async run(query: string, params?: Record): Promise { + if (params && Object.keys(params).length > 0) { + // Convert query to use ? placeholders and extract values in order + const paramKeys = Object.keys(params).sort() // Sort for consistent ordering + let processedQuery = query + const values: unknown[] = [] + + paramKeys.forEach(key => { + processedQuery = processedQuery.replace(new RegExp(`@${key}\\b`, 'g'), '?') + values.push(params[key]) + }) + + await sql.raw(processedQuery, values).execute(this.db) + } else { + await sql.raw(query).execute(this.db) + } + } + + async getColumns(tableName: string): Promise { + const result = await sql`select name from pragma_table_info(${tableName})`.execute(this.db) + return result.map((row: any) => row.name as string) + } + + async addColumns(tableName: string, newColumns: string[]): Promise { + for (const columnName of newColumns) { + await sql`ALTER TABLE ${sql.table(tableName)} ADD COLUMN ${sql.raw(columnName)}`.execute(this.db) + } + } + + async createTableNoTypes(tableName: string, columns: string[], noDrop: boolean = false): Promise { + const fields = uniq(columns.map(f => sanitise(f))) + if (!noDrop) { + await this.dropTable(tableName) + } + const fieldList = fields.join(',') + await sql`CREATE TABLE IF NOT EXISTS ${sql.table(tableName)}(${sql.raw(fieldList)})`.execute(this.db) + } + + async createTable(tableName: string, columns: ColumnDefinition[], noDrop: boolean = false): Promise { + const fields = uniqBy(columns.map(c => ({ + ...c, + name: sanitise(c.name) + })), 'name') + + if (!noDrop) { + await this.dropTable(tableName) + } + + const fieldDefinitions = fields.map(c => c.name) + const fieldList = fieldDefinitions.join(', ') + await sql`CREATE TABLE IF NOT EXISTS ${sql.table(tableName)}(${sql.raw(fieldList)})`.execute(this.db) + } + + async clearTable(tableName: string): Promise { + await sql`DELETE FROM ${sql.table(tableName)}`.execute(this.db) + } + + async insertData(tableName: string, data: Record[]): Promise { + if (data.length === 0) return + + for (const d of data) { + const formattedData = formatData(d) + const columns = Object.keys(formattedData).filter(c => c !== '__parsed_extra') + + if (columns.length === 0) continue + + // Use raw SQL for dynamic table names and columns + const columnNames = columns.join(', ') + const placeholders = columns.map(() => '?').join(', ') + const values = columns.map(col => formattedData[col]) + + await sql.raw( + `INSERT INTO ${tableName} (${columnNames}) VALUES (${placeholders})`, + values + ).execute(this.db) + } + } + + async dropTable(tableName: string): Promise { + await sql`DROP TABLE IF EXISTS ${sql.table(tableName)}`.execute(this.db) + } + + async select(query: string, params: Record): Promise<{ data: any[], columns: string[] }> { + // Convert query to use ? placeholders and extract values in order + const paramKeys = Object.keys(params).sort() // Sort for consistent ordering + let processedQuery = query + const values: unknown[] = [] + + paramKeys.forEach(key => { + processedQuery = processedQuery.replace(new RegExp(`@${key}\\b`, 'g'), '?') + values.push(params[key]) + }) + + const result = await sql.raw(processedQuery, values).execute(this.db) + const data = result.rows || result + + // Extract column names from the first row if available + const columns = Array.isArray(data) && data.length > 0 ? Object.keys(data[0]) : [] + + return { data: Array.isArray(data) ? data : [], columns } + } + + async updateData(tableName: string, data: Array>, matchKey: string = 'id'): Promise { + const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {})) + + for (const d of data) { + const formattedData = formatData(d) + const matchValue = formattedData[matchKey] + const updateData = { ...formattedData } + delete updateData[matchKey] + + if (Object.keys(updateData).length === 0) continue + + // Use raw SQL for dynamic updates + const setClause = Object.keys(updateData).map(key => `${key} = ?`).join(', ') + const values = [...Object.values(updateData), matchValue] + + await sql.raw( + `UPDATE ${tableName} SET ${setClause} WHERE ${matchKey} = ?`, + values + ).execute(this.db) + } + } + + async deleteData(name: string, data: Array>, key: string = 'id'): Promise { + for (const d of data) { + await sql.raw( + `DELETE FROM ${name} WHERE ${key} = ?`, + [d[key]] + ).execute(this.db) + } + } + + async connect(): Promise { + // Connection is handled by the DatabaseProvider, no-op here + return Promise.resolve() + } + + async explainQuery(query: string, params: Record): Promise { + // Convert query to use ? placeholders and extract values in order + const paramKeys = Object.keys(params).sort() // Sort for consistent ordering + let processedQuery = query + const values: unknown[] = [] + + paramKeys.forEach(key => { + processedQuery = processedQuery.replace(new RegExp(`@${key}\\b`, 'g'), '?') + values.push(params[key]) + }) + + const result = await sql.raw(`EXPLAIN QUERY PLAN ${processedQuery}`, values).execute(this.db) + return Array.isArray(result.rows) ? result.rows : Array.isArray(result) ? result : [] + } + + async disconnect(): Promise { + // Disconnection is handled by the DatabaseProvider, no-op here + return Promise.resolve() + } + + async createIndex(indexName: string, tableName: string, columns: string[]): Promise { + const columnList = columns.join(', ') + await sql`CREATE INDEX IF NOT EXISTS ${sql.raw(indexName)} ON ${sql.table(tableName)} (${sql.raw(columnList)})`.execute(this.db) + } +} \ No newline at end of file diff --git a/src/modules/database/sqlocal/queryDatabase.ts b/src/modules/database/sqlocal/queryDatabase.ts new file mode 100644 index 0000000..0319fab --- /dev/null +++ b/src/modules/database/sqlocal/queryDatabase.ts @@ -0,0 +1,53 @@ +import { BindingSpec, Database } from "@sqlite.org/sqlite-wasm"; + +interface Context { + filename: string; +} + +type ColumnTransformer = ( + db: Database, + context: Context, + query: string, +) => Promise; + +const TransformerId = async (_db: Database, context: Context, query: string) => + query; + +const transformBindings = (b: Record) => { + return Object.entries(b).reduce( + (acc, [key, value]) => ({ + ...acc, + [key.startsWith("?") ? key : `?${key}`]: value, + }), + {}, + ) as BindingSpec; +}; + +export class QueryDatabase { + constructor( + private db: Database, + private columnTransformer: ColumnTransformer = TransformerId, + ) {} + + async contextSelect( + query: string, + context: Context, + params: Record, + ) { + const q = await this.columnTransformer(this.db, context, query); + + const stmt = this.db.prepare(q).bind(transformBindings(params)); + + const results = []; + let columns = stmt.getColumnNames(); + while (stmt.step()) { + results.push(stmt.get({})); + } + stmt.finalize(); + + return { + data: results, + columns + } + } +} diff --git a/src/modules/database/sqlocal/schema.ts b/src/modules/database/sqlocal/schema.ts index d471a9e..52694d6 100644 --- a/src/modules/database/sqlocal/schema.ts +++ b/src/modules/database/sqlocal/schema.ts @@ -1,7 +1,40 @@ +export type DatabaseSchema = { + files: Files + links: Links + tags: Tags + tasks: Tasks +} + export type Files = { + id: string + name: string path: string + created_at: Date + modified_at: Date + file_size: number } -export type Database = { - files: Files +export type Links = { + path: string + target: string + position: string + display_text: string + target_exists: boolean +} + +export type Tags = { + tag: string + fileId: string + path: string +} + +export type Tasks = { + checkbox: string + task: string + completed: string + filePath: string + path: string + position: string + heading: string + heading_level: string } \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index a88d262..a09277c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,6 +12,7 @@ "isolatedModules": true, "skipLibCheck": true, "resolveJsonModule": true, + "strict": true, "types": [ "jest" ],