Skip to content

WIP: SQL Backend Migration #193

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
kulak-at wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

WIP: SQL Backend Migration #193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f94d97b
feat: WIP. Proof of concept of using sqlite-wasm
kulak-at Aug 11, 2025
cd46a77
chore: few experiments with kysely
kulak-at Aug 12, 2025
c3bcacc
Merge branch 'main' into feat/sql-backend-migration
kulak-at Sep 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions esbuild.config.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ if you want to view the source, please visit the github repository of this plugi
const wasmPlugin = {
name: 'wasm',
setup(build) {
// Handle .wasm files
build.onResolve({ filter: /\.wasm$/ }, args => {
if (args.resolveDir === '') return;
return {
Expand All @@ -37,6 +38,52 @@ const wasmPlugin = {
loader: 'js',
};
});

// Handle SQLite bundler-friendly module
build.onResolve({ filter: /sqlite3-bundler-friendly\.mjs$/ }, args => {
if (args.path.startsWith('@sqlite.org/sqlite-wasm')) {
// Resolve to the actual file path in node_modules
return {
path: join(process.cwd(), 'node_modules', args.path),
namespace: 'sqlite-bundler',
};
}
return {
path: join(args.resolveDir, args.path),
namespace: 'sqlite-bundler',
};
});

build.onLoad({ filter: /sqlite3-bundler-friendly\.mjs$/, namespace: 'sqlite-bundler' }, async (args) => {
const moduleContents = readFileSync(args.path, 'utf8');
const wasmPath = args.path.replace('sqlite3-bundler-friendly.mjs', 'sqlite3.wasm');
const wasmContents = readFileSync(wasmPath);
const wasmBase64 = wasmContents.toString('base64');

// Replace the findWasmBinary function to return embedded WASM
const modifiedContents = moduleContents
// Replace the entire findWasmBinary function
.replace(
/function\s+findWasmBinary\s*\(\s*\)\s*\{[\s\S]*?return\s+new\s+URL\s*\(\s*['"`]sqlite3\.wasm['"`]\s*,\s*import\.meta\.url\s*\)\.href\s*;?\s*\}/g,
`function findWasmBinary() {
return "data:application/wasm;base64,${wasmBase64}";
}`
)
// Also replace any direct new URL(...) patterns as fallback
.replace(
/new\s+URL\s*\(\s*['"`]sqlite3\.wasm['"`]\s*,\s*import\.meta\.url\s*\)\.href/g,
`"data:application/wasm;base64,${wasmBase64}"`
)
.replace(
/new\s+URL\s*\(\s*['"`]sqlite3\.wasm['"`]\s*,\s*import\.meta\.url\s*\)/g,
`"data:application/wasm;base64,${wasmBase64}"`
);

return {
contents: modifiedContents,
loader: 'js',
};
});
}
};

Expand Down
7 changes: 6 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
"@hypersphere/dity-graph": "^0.0.11",
"@hypersphere/omnibus": "^0.1.6",
"@jlongster/sql.js": "^1.6.7",
"@sqlite.org/sqlite-wasm": "3.50.3-build1",
"@types/jsonpath": "^0.2.4",
"@vanakat/plugin-api": "^0.2.1",
"absurd-sql": "^0.0.54",
Expand All @@ -70,6 +71,9 @@
"handlebars": "^4.7.8",
"json5": "^2.2.3",
"jsonpath": "^1.1.1",
"kysely": "^0.28.5",
"kysely-wasm": "^1.2.1",
"kysely-wasqlite-worker": "^1.2.1",
"lodash": "^4.17.21",
"markdown-table-ts": "^1.0.3",
"mermaid": "11.9.0",
Expand All @@ -78,6 +82,7 @@
"sql-parser-cst": "^0.33.1",
"unidecode": "^1.1.0",
"util": "^0.12.5",
"uuid": "^11.1.0"
"uuid": "^11.1.0",
"wa-sqlite": "^1.0.0"
}
}
74 changes: 74 additions & 0 deletions pnpm-lock.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 4 additions & 3 deletions src/main.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
provideGlobalGridOptions,
} from "ag-grid-community";
import { mainModule } from "./modules/main/module";
import { sql } from "kysely";

// Register all community features
ModuleRegistry.registerModules([AllCommunityModule]);
Expand All @@ -14,7 +15,7 @@ ModuleRegistry.registerModules([AllCommunityModule]);
provideGlobalGridOptions({ theme: "legacy" });

export default class SqlSealPlugin extends Plugin {
container: ReturnType<(typeof mainModule)["build"]>;
container?: ReturnType<(typeof mainModule)["build"]>;

async onload() {
// CONTAINER
Expand All @@ -24,7 +25,7 @@ export default class SqlSealPlugin extends Plugin {
.resolve('obsidian.vault', d => d.value(this.app.vault))
.build()

const init = await this.container.get("init")
init()
const init = await this.container.get("init");
init();
}
}
9 changes: 7 additions & 2 deletions src/modules/database/module.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
import { Registrator } from "@hypersphere/dity";
import { App } from "obsidian";
import { DatabaseProvider } from "./sqlocal/databaseProvider";
import { Registrator } from "@hypersphere/dity";
import { databaseFactory } from "./factory";


export type DatabaseModule = typeof db

export const db = new Registrator()
.import<'app', App>()
.register('db', d => d.fn(databaseFactory).inject('app'))
.export('db')
.register('provider', d => d.cls(DatabaseProvider).inject('app'))
.export('db', 'provider')
110 changes: 110 additions & 0 deletions src/modules/database/sqlocal/databaseProvider.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
import { Kysely } from "kysely";
import { DatabaseSchema } from "./schema";
import { OfficialWasmDialect } from "kysely-wasm";
import sqlite3InitModule from "@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3-bundler-friendly.mjs";

Check failure on line 4 in src/modules/database/sqlocal/databaseProvider.ts

View workflow job for this annotation

GitHub Actions / typecheck 

Could not find a declaration file for module '@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3-bundler-friendly.mjs'. '/home/runner/work/sql-seal/sql-seal/node_modules/.pnpm/@sqlite.org+sqlite-wasm@3.50.3-build1/node_modules/@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3-bundler-friendly.mjs' implicitly has an 'any' type.
import { Database } from '@sqlite.org/sqlite-wasm';
import { App } from "obsidian";
import { sanitise } from "../../../utils/sanitiseColumn";

interface DbMapEntry {
database: any
kysely: Kysely<any>
}

export class DatabaseProvider {
constructor(private app: App) {}
private databases: Map<string, DbMapEntry> = new Map();

private _sqlite3: any = null;
private async sqlite3() {
if (this._sqlite3) {
return this._sqlite3;
}
this._sqlite3 = (await sqlite3InitModule()).oo1;
return this._sqlite3;
}

get prefix() {
const filename =
sanitise(this.app.vault.getName()) + "___" + (this.app as any).appId;
return filename;
}

async get<F extends string | null>(
filename: F,
): Promise<F extends null ? Kysely<DatabaseSchema> : Kysely<unknown>> {
const key = filename ?? "GLOBAL";

const db = this.databases.get(key);
if (db) {
return db as any; // FIXME: fix typing here
}

const path = this.prefix + "___" + key;

let rawDatabase: Database | null = null

try {
// Create dialect using official SQLite WASM
const dialect = new OfficialWasmDialect({
database: async () => {
const sqlite3 = await this.sqlite3();
if (!sqlite3) {
throw new Error("Failed to load SQLite WASM");
}
const dbPath = path;

const db = new sqlite3.DB(dbPath, "ct");
rawDatabase = db
return db
},
});


// Create Kysely instance
const kysely = new Kysely<any>({
dialect,
});

this.databases.set(key, {
kysely,
database: rawDatabase
});
return db as any;
} catch (error) {
console.error("Error creating SQLite WASM database:", error);
throw error;
}
}

async getGlobal() {
const db = await this.get(null)
const tables = await db.introspection.getTables()
if (!tables.length) {
// Generating schema

}
}

async getRawDatabaseAccess(filename: string | null = null) {
await this.get(filename)
return this.databases.get(filename ?? 'GLOBAL')!.database
}

createGlobalTables(db: Kysely<any>) {
db.schema
.createTable('files')
.addColumn('id', 'text')
.addColumn('name', 'text')
.addColumn('path', 'text')
.addColumn('created_at', 'datetime', c => c.defaultTo('now'))
.addColumn('modified_at', 'datetime')
.addColumn('file_size', 'numeric')
}

async close() {
return Promise.all(
Array.from(this.databases.entries()).map(([_, db]) => db.destroy()),

Check failure on line 107 in src/modules/database/sqlocal/databaseProvider.ts

View workflow job for this annotation

GitHub Actions / typecheck 

Property 'destroy' does not exist on type 'DbMapEntry'.
);
}
}
Loading