From 811205dbf22677f64c3f08c9a99a666b6ae16c0e Mon Sep 17 00:00:00 2001 From: tdrz Date: Wed, 18 Jun 2025 10:35:04 +0200 Subject: [PATCH 01/79] initial commit --- packages/pglite/src/pglite.ts | 205 +++++++++++++++++++--------- packages/pglite/src/postgresMod.ts | 3 +- packages/pglite/tests/basic.test.ts | 2 +- packages/pglite/tests/clone.test.js | 4 +- 4 files changed, 142 insertions(+), 72 deletions(-) diff --git a/packages/pglite/src/pglite.ts b/packages/pglite/src/pglite.ts index 32ee243a0..ff3a6f2a0 100644 --- a/packages/pglite/src/pglite.ts +++ b/packages/pglite/src/pglite.ts @@ -77,6 +77,15 @@ export class PGlite #notifyListeners = new Map void>>() #globalNotifyListeners = new Set<(channel: string, payload: string) => void>() + // receive data from wasm + #onWriteDataPtr: number = -1 + #accumulatedData: any = [] + + // send data to wasm + #onReadDataPtr: number = -1 + #outputData: any = [] + #outputDataReadBytes: number = 0 + /** * Create a new PGlite instance * @param dataDir The directory to store the database files @@ -130,6 +139,8 @@ export class PGlite this.#dataTransferContainer = options.defaultDataTransferContainer } + console.log(this.#dataTransferContainer) + // Save the extensions for later use this.#extensions = options.extensions ?? {} @@ -369,6 +380,56 @@ export class PGlite // Load the database engine this.mod = await PostgresModFactory(emscriptenOpts) + // set the write callback + this.#onWriteDataPtr = (this.mod as any).addFunction((ptr: any, length: number) => { + let bytes + try { + bytes = this.mod!.HEAPU8.subarray(ptr, ptr + length) + } catch (e: any) { + console.error('error', e) + throw e + } + // const bytes = new Uint8Array(this.mod!.HEAPU8, data, length); // View into WASM memory + let copied = bytes.slice(); + + try { + + // Append copied data to accumulatedData + if (!this.#accumulatedData) { + console.error('no accumulatedData') + } + if (this.#accumulatedData.length < 0) { + console.error('length less than 0') + } + const newAccumulated = new Uint8Array(this.#accumulatedData.length + copied.length); + newAccumulated.set(this.#accumulatedData, 0); + newAccumulated.set(copied, this.#accumulatedData.length); + this.#accumulatedData = newAccumulated; + } catch (e) { + console.log(e) + } + return this.#accumulatedData.length + }, 'iii'); + + // set the read callback + this.#onReadDataPtr = (this.mod as any).addFunction((ptr: any, max_length: number) => { + // copy current data to wasm buffer + let length = this.#outputData.length + if (this.#outputData.length - this.#outputDataReadBytes > max_length) { + length = max_length + } + try { + + this.mod!.HEAP8.set((this.#outputData as Uint8Array).subarray(this.#outputDataReadBytes, this.#outputDataReadBytes + length), ptr); + this.#outputDataReadBytes += length + } catch (e) { + console.log(e) + } + return length + }, 'iii'); + + this.mod._set_read_write_cbs(this.#onReadDataPtr, this.#onWriteDataPtr) + // Sync the filesystem from any previous store await this.fs!.initialSyncFs() @@ -578,84 +639,92 @@ export class PGlite message: Uint8Array, options: { dataTransferContainer?: DataTransferContainer } = {}, ) { - let data + // let data const mod = this.mod! - + console.log(options) // >0 set buffer content type to wire protocol mod._use_wire(1) - const msg_len = message.length + // const msg_len = message.length // TODO: if (message.length>CMA_B) force file - let currDataTransferContainer = - options.dataTransferContainer ?? this.#dataTransferContainer + // let currDataTransferContainer = + // options.dataTransferContainer ?? this.#dataTransferContainer // do we overflow allocated shared memory segment - if (message.length >= mod.FD_BUFFER_MAX) currDataTransferContainer = 'file' - - switch (currDataTransferContainer) { - case 'cma': { - // set buffer size so answer will be at size+0x2 pointer addr - mod._interactive_write(message.length) - // TODO: make it seg num * seg maxsize if multiple channels. - mod.HEAPU8.set(message, 1) - break - } - case 'file': { - // Use socketfiles to emulate a socket connection - const pg_lck = '/tmp/pglite/base/.s.PGSQL.5432.lck.in' - const pg_in = '/tmp/pglite/base/.s.PGSQL.5432.in' - mod._interactive_write(0) - mod.FS.writeFile(pg_lck, message) - mod.FS.rename(pg_lck, pg_in) - break - } - default: - throw new Error( - `Unknown data transfer container: ${currDataTransferContainer}`, - ) - } + // if (message.length >= mod.FD_BUFFER_MAX) currDataTransferContainer = 'file' + + // switch (currDataTransferContainer) { + // case 'cma': { + // // set buffer size so answer will be at size+0x2 pointer addr + // mod._interactive_write(message.length) + // // TODO: make it seg num * seg maxsize if multiple channels. + // mod.HEAPU8.set(message, 1) + // break + // } + // case 'file': { + // // Use socketfiles to emulate a socket connection + // const pg_lck = '/tmp/pglite/base/.s.PGSQL.5432.lck.in' + // const pg_in = '/tmp/pglite/base/.s.PGSQL.5432.in' + // mod._interactive_write(0) + // mod.FS.writeFile(pg_lck, message) + // mod.FS.rename(pg_lck, pg_in) + // break + // } + // default: + // throw new Error( + // `Unknown data transfer container: ${currDataTransferContainer}`, + // ) + // } + + this.#accumulatedData = [] + this.#outputDataReadBytes = 0 + this.#outputData = message // execute the message - mod._interactive_one() - - const channel = mod._get_channel() - if (channel < 0) currDataTransferContainer = 'file' - - // TODO: use channel value for msg_start - if (channel > 0) currDataTransferContainer = 'cma' - - switch (currDataTransferContainer) { - case 'cma': { - // Read responses from the buffer - - const msg_start = msg_len + 2 - const msg_end = msg_start + mod._interactive_read() - data = mod.HEAPU8.subarray(msg_start, msg_end) - break - } - case 'file': { - // Use socketfiles to emulate a socket connection - const pg_out = '/tmp/pglite/base/.s.PGSQL.5432.out' - try { - const fstat = mod.FS.stat(pg_out) - const stream = mod.FS.open(pg_out, 'r') - data = new Uint8Array(fstat.size) - mod.FS.read(stream, data, 0, fstat.size, 0) - mod.FS.unlink(pg_out) - } catch (x) { - // case of single X message. - data = new Uint8Array(0) - } - break - } - default: - throw new Error( - `Unknown data transfer container: ${currDataTransferContainer}`, - ) - } - - return data + mod._interactive_one(message.length, message[0]) + + this.#outputData = [] + + // const channel = mod._get_channel() + // if (channel < 0) currDataTransferContainer = 'file' + + // // TODO: use channel value for msg_start + // if (channel > 0) currDataTransferContainer = 'cma' + + // switch (currDataTransferContainer) { + // case 'cma': { + // // Read responses from the buffer + + // const msg_start = msg_len + 2 + // const msg_end = msg_start + mod._interactive_read() + // data = mod.HEAPU8.subarray(msg_start, msg_end) + // break + // } + // case 'file': { + // // Use socketfiles to emulate a socket connection + // const pg_out = '/tmp/pglite/base/.s.PGSQL.5432.out' + // try { + // const fstat = mod.FS.stat(pg_out) + // const stream = mod.FS.open(pg_out, 'r') + // data = new Uint8Array(fstat.size) + // mod.FS.read(stream, data, 0, fstat.size, 0) + // mod.FS.unlink(pg_out) + // } catch (x) { + // // case of single X message. + // data = new Uint8Array(0) + // } + // break + // } + // default: + // throw new Error( + // `Unknown data transfer container: ${currDataTransferContainer}`, + // ) + // } + + // return data + if (this.#accumulatedData.length) return this.#accumulatedData + return new Uint8Array(0) } /** diff --git a/packages/pglite/src/postgresMod.ts b/packages/pglite/src/postgresMod.ts index abb4a0155..9cd8887f0 100644 --- a/packages/pglite/src/postgresMod.ts +++ b/packages/pglite/src/postgresMod.ts @@ -32,8 +32,9 @@ export interface PostgresMod _get_buffer_addr: (fd: number) => number _get_channel: () => number _interactive_write: (msgLength: number) => void - _interactive_one: () => void + _interactive_one: (length: number, peek: number) => void _interactive_read: () => number + _set_read_write_cbs: (read_cb: number, write_cb: number) => void } type PostgresFactory = ( diff --git a/packages/pglite/tests/basic.test.ts b/packages/pglite/tests/basic.test.ts index f7a4040db..c9d88c9af 100644 --- a/packages/pglite/tests/basic.test.ts +++ b/packages/pglite/tests/basic.test.ts @@ -12,7 +12,7 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { describe(`basic`, () => { it('exec', async () => { - const db = new PGlite({ + const db = await PGlite.create({ defaultDataTransferContainer, }) await db.exec(` diff --git a/packages/pglite/tests/clone.test.js b/packages/pglite/tests/clone.test.js index 81283f660..d33824ef2 100644 --- a/packages/pglite/tests/clone.test.js +++ b/packages/pglite/tests/clone.test.js @@ -3,7 +3,7 @@ import { PGlite } from '../dist/index.js' describe('clone', () => { it('clone pglite instance', async () => { - const pg1 = new PGlite() + const pg1 = await PGlite.create() await pg1.exec(` CREATE TABLE IF NOT EXISTS test ( id SERIAL PRIMARY KEY, @@ -21,7 +21,7 @@ describe('clone', () => { }) it('clone pglite instance - insert into pg2', async () => { - const pg1 = new PGlite() + const pg1 = await PGlite.create() await pg1.exec(` CREATE TABLE IF NOT EXISTS test ( id SERIAL PRIMARY KEY, From 3385c69597f4fde103cf6bbc21cd0c5ee9a0ac3f Mon Sep 17 00:00:00 2001 From: tdrz Date: Wed, 18 Jun 2025 15:23:51 +0200 Subject: [PATCH 02/79] bug fixes; cleanup; most pglite tests pass --- packages/benchmark/src/rtt.js | 2 - packages/pglite/src/base.ts | 2 +- packages/pglite/src/interface.ts | 4 - packages/pglite/src/pglite.ts | 141 +++++----- packages/pglite/src/worker/index.ts | 8 +- packages/pglite/tests/basic.test.ts | 43 +-- packages/pglite/tests/exec-protocol.test.ts | 103 ++++--- packages/pglite/tests/live.test.ts | 20 +- packages/pglite/tests/notify.test.ts | 292 ++++++++++---------- packages/pglite/tests/pg_ivm.test.ts | 8 +- packages/pglite/tests/pgvector.test.ts | 6 +- packages/pglite/tests/query-sizes.test.ts | 139 +++++----- packages/pglite/tests/test-utils.ts | 31 +-- packages/pglite/tests/user.test.ts | 160 +++++------ 14 files changed, 428 insertions(+), 531 deletions(-) diff --git a/packages/benchmark/src/rtt.js b/packages/benchmark/src/rtt.js index c9cc362b4..421113647 100644 --- a/packages/benchmark/src/rtt.js +++ b/packages/benchmark/src/rtt.js @@ -9,13 +9,11 @@ const CONFIGURATIONS = new Map( label: 'PGlite Memory
(CMA Transport default)', db: 'pglite', dataDir: '', - options: { defaultDataTransferContainer: 'cma' }, }, { label: 'PGlite Memory
(File Transport)', db: 'pglite', dataDir: '', - options: { defaultDataTransferContainer: 'file' }, }, { label: 'PGlite IDB', diff --git a/packages/pglite/src/base.ts b/packages/pglite/src/base.ts index 07cc43a09..b21e956a8 100644 --- a/packages/pglite/src/base.ts +++ b/packages/pglite/src/base.ts @@ -63,7 +63,7 @@ export abstract class BasePGlite */ abstract execProtocolRaw( message: Uint8Array, - { syncToFs, dataTransferContainer }: ExecProtocolOptions, + { syncToFs }: ExecProtocolOptions, ): Promise /** diff --git a/packages/pglite/src/interface.ts b/packages/pglite/src/interface.ts index 0ca50c09e..82866093b 100644 --- a/packages/pglite/src/interface.ts +++ b/packages/pglite/src/interface.ts @@ -33,7 +33,6 @@ export interface ExecProtocolOptions { syncToFs?: boolean throwOnError?: boolean onNotice?: (notice: NoticeMessage) => void - dataTransferContainer?: DataTransferContainer } export interface ExtensionSetupResult { @@ -78,8 +77,6 @@ export interface DumpDataDirResult { filename: string } -export type DataTransferContainer = 'cma' | 'file' - export interface PGliteOptions { dataDir?: string username?: string @@ -94,7 +91,6 @@ export interface PGliteOptions { fsBundle?: Blob | File parsers?: ParserOptions serializers?: SerializerOptions - defaultDataTransferContainer?: DataTransferContainer } export type PGliteInterface = diff --git a/packages/pglite/src/pglite.ts b/packages/pglite/src/pglite.ts index ff3a6f2a0..0bbb103d5 100644 --- a/packages/pglite/src/pglite.ts +++ b/packages/pglite/src/pglite.ts @@ -17,7 +17,6 @@ import type { PGliteInterface, PGliteInterfaceExtensions, PGliteOptions, - DataTransferContainer, } from './interface.js' import PostgresModFactory, { type PostgresMod } from './postgresMod.js' import { @@ -60,8 +59,6 @@ export class PGlite #fsSyncMutex = new Mutex() #fsSyncScheduled = false - #dataTransferContainer: DataTransferContainer = 'cma' - readonly debug: DebugLevel = 0 #extensions: Extensions @@ -77,14 +74,21 @@ export class PGlite #notifyListeners = new Map void>>() #globalNotifyListeners = new Set<(channel: string, payload: string) => void>() + static readonly RECV_BUF_SIZE: number = 10 * 1024 * 1024 // 10MB default + // receive data from wasm #onWriteDataPtr: number = -1 - #accumulatedData: any = [] + // buffer that holds data received from wasm + #inputData = new Uint8Array(PGlite.RECV_BUF_SIZE) + // write index in the buffer + #writeOffset: number = 0 // send data to wasm #onReadDataPtr: number = -1 + // buffer that sends the data to be sent to wasm #outputData: any = [] - #outputDataReadBytes: number = 0 + // read index in the buffer + #readOffset: number = 0 /** * Create a new PGlite instance @@ -134,13 +138,6 @@ export class PGlite this.#relaxedDurability = options.relaxedDurability } - // Set the default data transfer container - if (options?.defaultDataTransferContainer !== undefined) { - this.#dataTransferContainer = options.defaultDataTransferContainer - } - - console.log(this.#dataTransferContainer) - // Save the extensions for later use this.#extensions = options.extensions ?? {} @@ -381,52 +378,67 @@ export class PGlite this.mod = await PostgresModFactory(emscriptenOpts) // set the write callback - this.#onWriteDataPtr = (this.mod as any).addFunction((ptr: any, length: number) => { - let bytes - try { - bytes = this.mod!.HEAPU8.subarray(ptr, ptr + length) - } catch (e: any) { - console.error('error', e) - throw e - } - // const bytes = new Uint8Array(this.mod!.HEAPU8, data, length); // View into WASM memory - let copied = bytes.slice(); - - try { - - // Append copied data to accumulatedData - if (!this.#accumulatedData) { - console.error('no accumulatedData') + this.#onWriteDataPtr = (this.mod as any).addFunction( + (ptr: any, length: number) => { + let bytes + try { + bytes = this.mod!.HEAPU8.subarray(ptr, ptr + length) + } catch (e: any) { + console.error('error', e) + throw e } - if (this.#accumulatedData.length < 0) { - console.error('length less than 0') + const copied = bytes.slice() + + const requiredSize = this.#writeOffset + copied.length + + if (requiredSize > this.#inputData.length) { + // Expand buffer size (double until it fits) + let newSize = this.#inputData.length + while (newSize < requiredSize) { + newSize *= 2 + } + + const newBuffer = new Uint8Array(newSize) + newBuffer.set(this.#inputData.subarray(0, this.#writeOffset)) + this.#inputData = newBuffer } - const newAccumulated = new Uint8Array(this.#accumulatedData.length + copied.length); - newAccumulated.set(this.#accumulatedData, 0); - newAccumulated.set(copied, this.#accumulatedData.length); - this.#accumulatedData = newAccumulated; - } catch (e) { - console.log(e) - } - return this.#accumulatedData.length - }, 'iii'); - // set the read callback - this.#onReadDataPtr = (this.mod as any).addFunction((ptr: any, max_length: number) => { - // copy current data to wasm buffer - let length = this.#outputData.length - if (this.#outputData.length - this.#outputDataReadBytes > max_length) { - length = max_length - } - try { + this.#inputData.set(copied, this.#writeOffset) + this.#writeOffset += copied.length - this.mod!.HEAP8.set((this.#outputData as Uint8Array).subarray(this.#outputDataReadBytes, this.#outputDataReadBytes + length), ptr); - this.#outputDataReadBytes += length - } catch (e) { - console.log(e) - } - return length - }, 'iii'); + // const newAccumulated = new Uint8Array(this.#accumulatedData.length + copied.length); + // newAccumulated.set(this.#accumulatedData, 0); + // newAccumulated.set(copied, this.#accumulatedData.length); + // this.#accumulatedData = newAccumulated; + return this.#inputData.length + }, + 'iii', + ) + + // set the read callback + this.#onReadDataPtr = (this.mod as any).addFunction( + (ptr: any, max_length: number) => { + // copy current data to wasm buffer + let length = this.#outputData.length - this.#readOffset + if (length > max_length) { + length = max_length + } + try { + this.mod!.HEAP8.set( + (this.#outputData as Uint8Array).subarray( + this.#readOffset, + this.#readOffset + length, + ), + ptr, + ) + this.#readOffset += length + } catch (e) { + console.log(e) + } + return length + }, + 'iii', + ) this.mod._set_read_write_cbs(this.#onReadDataPtr, this.#onWriteDataPtr) @@ -635,13 +647,9 @@ export class PGlite * @param message The postgres wire protocol message to execute * @returns The direct message data response produced by Postgres */ - execProtocolRawSync( - message: Uint8Array, - options: { dataTransferContainer?: DataTransferContainer } = {}, - ) { + execProtocolRawSync(message: Uint8Array) { // let data const mod = this.mod! - console.log(options) // >0 set buffer content type to wire protocol mod._use_wire(1) // const msg_len = message.length @@ -677,10 +685,15 @@ export class PGlite // ) // } - this.#accumulatedData = [] - this.#outputDataReadBytes = 0 + // this.#accumulatedData = [] + if (this.#inputData.buffer.byteLength > PGlite.RECV_BUF_SIZE) { + this.#inputData = new Uint8Array(PGlite.RECV_BUF_SIZE) + } + this.#readOffset = 0 this.#outputData = message + this.#writeOffset = 0 + // execute the message mod._interactive_one(message.length, message[0]) @@ -723,7 +736,7 @@ export class PGlite // } // return data - if (this.#accumulatedData.length) return this.#accumulatedData + if (this.#writeOffset) return this.#inputData.subarray(0, this.#writeOffset) return new Uint8Array(0) } @@ -740,9 +753,9 @@ export class PGlite */ async execProtocolRaw( message: Uint8Array, - { syncToFs = true, dataTransferContainer }: ExecProtocolOptions = {}, + { syncToFs = true }: ExecProtocolOptions = {}, ) { - const data = this.execProtocolRawSync(message, { dataTransferContainer }) + const data = this.execProtocolRawSync(message) if (syncToFs) { await this.syncToFs() } diff --git a/packages/pglite/src/worker/index.ts b/packages/pglite/src/worker/index.ts index 8148824b1..f28fa0b77 100644 --- a/packages/pglite/src/worker/index.ts +++ b/packages/pglite/src/worker/index.ts @@ -1,5 +1,4 @@ import type { - DataTransferContainer, DebugLevel, ExecProtocolResult, Extensions, @@ -632,11 +631,8 @@ function makeWorkerApi(tabId: string, db: PGlite) { return { messages, data } } }, - async execProtocolRaw( - message: Uint8Array, - options: { dataTransferContainer?: DataTransferContainer } = {}, - ) { - const result = await db.execProtocolRaw(message, options) + async execProtocolRaw(message: Uint8Array) { + const result = await db.execProtocolRaw(message) if (result.byteLength !== result.buffer.byteLength) { // The data is a slice of a larger buffer, this is potentially the whole // memory of the WASM module. We copy it to a new Uint8Array and return that. diff --git a/packages/pglite/tests/basic.test.ts b/packages/pglite/tests/basic.test.ts index c9d88c9af..eec310615 100644 --- a/packages/pglite/tests/basic.test.ts +++ b/packages/pglite/tests/basic.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect } from 'vitest' import { expectToThrowAsync, testEsmCjsAndDTC } from './test-utils.ts' import { identifier } from '../dist/templating.js' -await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { +await testEsmCjsAndDTC(async (importType) => { const { PGlite } = importType === 'esm' ? await import('../dist/index.js') @@ -12,9 +12,7 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { describe(`basic`, () => { it('exec', async () => { - const db = await PGlite.create({ - defaultDataTransferContainer, - }) + const db = await PGlite.create() await db.exec(` CREATE TABLE IF NOT EXISTS test ( id SERIAL PRIMARY KEY, @@ -51,9 +49,7 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { }) it('query', async () => { - const db = new PGlite({ - defaultDataTransferContainer, - }) + const db = new PGlite() await db.query(` CREATE TABLE IF NOT EXISTS test ( id SERIAL PRIMARY KEY, @@ -94,9 +90,7 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { }) it('query templated', async () => { - const db = new PGlite({ - defaultDataTransferContainer, - }) + const db = new PGlite() const tableName = identifier`test` await db.sql` CREATE TABLE IF NOT EXISTS ${tableName} ( @@ -137,9 +131,7 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { }) it('types', async () => { - const db = new PGlite({ - defaultDataTransferContainer, - }) + const db = new PGlite() await db.query(` CREATE TABLE IF NOT EXISTS test ( id SERIAL PRIMARY KEY, @@ -301,7 +293,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('custom parser and serializer', async () => { const db = new PGlite({ - defaultDataTransferContainer, serializers: { 1700: (x) => x.toString() }, parsers: { 1700: (x) => BigInt(x) }, }) @@ -338,9 +329,7 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { }) it('params', async () => { - const db = new PGlite({ - defaultDataTransferContainer, - }) + const db = new PGlite() await db.query(` CREATE TABLE IF NOT EXISTS test ( id SERIAL PRIMARY KEY, @@ -374,9 +363,7 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { }) it('array params', async () => { - const db = new PGlite({ - defaultDataTransferContainer, - }) + const db = new PGlite() await db.query(` CREATE TABLE IF NOT EXISTS test ( id SERIAL PRIMARY KEY, @@ -429,9 +416,7 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { }) it('error', async () => { - const db = new PGlite({ - defaultDataTransferContainer, - }) + const db = new PGlite() await expectToThrowAsync(async () => { await db.query('SELECT * FROM test;') }, 'relation "test" does not exist') @@ -501,9 +486,7 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { }) it('copy to/from blob', async () => { - const db = new PGlite({ - defaultDataTransferContainer, - }) + const db = new PGlite() await db.exec(` CREATE TABLE IF NOT EXISTS test ( id SERIAL PRIMARY KEY, @@ -571,9 +554,7 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { }) it('close', async () => { - const db = new PGlite({ - defaultDataTransferContainer, - }) + const db = new PGlite() await db.query(` CREATE TABLE IF NOT EXISTS test ( id SERIAL PRIMARY KEY, @@ -588,9 +569,7 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { }) it('use same param multiple times', async () => { - const db = new PGlite({ - defaultDataTransferContainer, - }) + const db = new PGlite() await db.exec(` CREATE TABLE IF NOT EXISTS test ( diff --git a/packages/pglite/tests/exec-protocol.test.ts b/packages/pglite/tests/exec-protocol.test.ts index 32e2d2e36..530694a1b 100644 --- a/packages/pglite/tests/exec-protocol.test.ts +++ b/packages/pglite/tests/exec-protocol.test.ts @@ -1,71 +1,66 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest' import { PGlite } from '../dist/index.js' import { serialize } from '@electric-sql/pg-protocol' -import { testDTC } from './test-utils.js' -testDTC(async (defaultDataTransferContainer) => { - describe('exec protocol', () => { - let db: PGlite +describe('exec protocol', () => { + let db: PGlite - beforeAll(async () => { - db = await PGlite.create({ - defaultDataTransferContainer, - }) - }) + beforeAll(async () => { + db = await PGlite.create() + }) - afterAll(async () => { - await db.close() - }) + afterAll(async () => { + await db.close() + }) - it('should perform a simple query', async () => { - const result = await db.execProtocol(serialize.query('SELECT 1')) - const messageNames = result.messages.map((msg) => msg.name) - expect(messageNames).toEqual([ - 'rowDescription', - 'dataRow', - 'commandComplete', - 'readyForQuery', - ]) - }) + it('should perform a simple query', async () => { + const result = await db.execProtocol(serialize.query('SELECT 1')) + const messageNames = result.messages.map((msg) => msg.name) + expect(messageNames).toEqual([ + 'rowDescription', + 'dataRow', + 'commandComplete', + 'readyForQuery', + ]) + }) - it('should perform an extended query', async () => { - const r1 = await db.execProtocol(serialize.parse({ text: 'SELECT $1' })) - const messageNames1 = r1.messages.map((msg) => msg.name) - expect(messageNames1).toEqual([ - 'notice', - 'parseComplete', - /* 'readyForQuery',*/ - ]) + it('should perform an extended query', async () => { + const r1 = await db.execProtocol(serialize.parse({ text: 'SELECT $1' })) + const messageNames1 = r1.messages.map((msg) => msg.name) + expect(messageNames1).toEqual([ + 'notice', + 'parseComplete', + /* 'readyForQuery',*/ + ]) - const r2 = await db.execProtocol(serialize.bind({ values: ['1'] })) - const messageNames2 = r2.messages.map((msg) => msg.name) - expect(messageNames2).toEqual(['notice', 'bindComplete']) + const r2 = await db.execProtocol(serialize.bind({ values: ['1'] })) + const messageNames2 = r2.messages.map((msg) => msg.name) + expect(messageNames2).toEqual(['notice', 'bindComplete']) - const r3 = await db.execProtocol(serialize.describe({ type: 'P' })) - const messageNames3 = r3.messages.map((msg) => msg.name) - expect(messageNames3).toEqual(['rowDescription']) + const r3 = await db.execProtocol(serialize.describe({ type: 'P' })) + const messageNames3 = r3.messages.map((msg) => msg.name) + expect(messageNames3).toEqual(['rowDescription']) - const r4 = await db.execProtocol(serialize.execute({})) - const messageNames4 = r4.messages.map((msg) => msg.name) - expect(messageNames4).toEqual(['dataRow', 'commandComplete']) + const r4 = await db.execProtocol(serialize.execute({})) + const messageNames4 = r4.messages.map((msg) => msg.name) + expect(messageNames4).toEqual(['dataRow', 'commandComplete']) - const r5 = await db.execProtocol(serialize.sync()) - const messageNames5 = r5.messages.map((msg) => msg.name) - expect(messageNames5).toEqual(['readyForQuery']) - }) + const r5 = await db.execProtocol(serialize.sync()) + const messageNames5 = r5.messages.map((msg) => msg.name) + expect(messageNames5).toEqual(['readyForQuery']) + }) - it('should handle error', async () => { - const result = await db.execProtocol(serialize.query('invalid sql'), { - throwOnError: false, - }) - const messageNames = result.messages.map((msg) => msg.name) - expect(messageNames).toEqual(['error', 'readyForQuery']) + it('should handle error', async () => { + const result = await db.execProtocol(serialize.query('invalid sql'), { + throwOnError: false, }) + const messageNames = result.messages.map((msg) => msg.name) + expect(messageNames).toEqual(['error', 'readyForQuery']) + }) - it('should throw error', async () => { - await expect( - db.execProtocol(serialize.query('invalid sql')), - ).rejects.toThrow() - }) + it('should throw error', async () => { + await expect( + db.execProtocol(serialize.query('invalid sql')), + ).rejects.toThrow() }) }) diff --git a/packages/pglite/tests/live.test.ts b/packages/pglite/tests/live.test.ts index 2c0fd4774..43c43ce59 100644 --- a/packages/pglite/tests/live.test.ts +++ b/packages/pglite/tests/live.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from 'vitest' import { testEsmCjsAndDTC } from './test-utils.ts' -await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { +await testEsmCjsAndDTC(async (importType) => { const { PGlite } = ( importType === 'esm' ? await import('../dist/index.js') @@ -17,7 +17,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('basic live query', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -113,7 +112,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('live query on view', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -224,7 +222,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('live query with params', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -310,7 +307,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('incremental query unordered', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -360,7 +356,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('incremental query with non-integer key', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -410,7 +405,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('basic live incremental query', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -507,7 +501,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('basic live incremental query with limit 1', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -550,7 +543,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('live incremental query on view', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -662,7 +654,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('live incremental query with params', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -749,7 +740,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('basic live changes', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -925,7 +915,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('subscribe to live query after creation', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -982,7 +971,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('live changes limit 1', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -1049,7 +1037,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('subscribe to live changes after creation', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -1109,7 +1096,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('live query with windowing', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -1201,7 +1187,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('throws error when only one of offset/limit is provided', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await expect( @@ -1244,7 +1229,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it("doesn't have a race condition when unsubscribing from a live query", async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -1292,7 +1276,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('works with pattern matching', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` @@ -1348,7 +1331,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { it('basic live query - case sensitive table name', async () => { const db = await PGlite.create({ extensions: { live }, - defaultDataTransferContainer, }) await db.exec(` diff --git a/packages/pglite/tests/notify.test.ts b/packages/pglite/tests/notify.test.ts index 7164255cb..001b947d7 100644 --- a/packages/pglite/tests/notify.test.ts +++ b/packages/pglite/tests/notify.test.ts @@ -1,185 +1,173 @@ import { describe, it, expect, vi } from 'vitest' import { PGlite } from '../dist/index.js' -import { expectToThrowAsync, testDTC } from './test-utils.js' +import { expectToThrowAsync } from './test-utils.js' -testDTC(async (defaultDataTransferContainer) => { - describe('notify API', () => { - it('notify', async () => { - const db = new PGlite({ - defaultDataTransferContainer, - }) +describe('notify API', () => { + it('notify', async () => { + const db = new PGlite() - await db.listen('test', (payload) => { - expect(payload).toBe('321') - }) + await db.listen('test', (payload) => { + expect(payload).toBe('321') + }) + + await db.exec("NOTIFY test, '321'") + + await new Promise((resolve) => setTimeout(resolve, 1000)) + }) - await db.exec("NOTIFY test, '321'") + it('unlisten', async () => { + const db = new PGlite() - await new Promise((resolve) => setTimeout(resolve, 1000)) + const unsub = await db.listen('test', () => { + throw new Error('Notification received after unsubscribed') }) - it('unlisten', async () => { - const db = new PGlite({ - defaultDataTransferContainer, - }) + await unsub() - const unsub = await db.listen('test', () => { - throw new Error('Notification received after unsubscribed') - }) + await db.exec('NOTIFY test') - await unsub() + await new Promise((resolve) => setTimeout(resolve, 1000)) + }) - await db.exec('NOTIFY test') + it('onNotification', async () => { + const db = new PGlite() - await new Promise((resolve) => setTimeout(resolve, 1000)) + db.onNotification((chan, payload) => { + expect(chan).toBe('test') + expect(payload).toBe('123') }) - it('onNotification', async () => { - const db = new PGlite({ - defaultDataTransferContainer, - }) + await db.exec('LISTEN test') + await db.exec("NOTIFY test, '123'") + + await new Promise((resolve) => setTimeout(resolve, 1000)) + }) + + it('check notify case sensitivity + special chars as Postgresql', async () => { + const pg = new PGlite() - db.onNotification((chan, payload) => { - expect(chan).toBe('test') - expect(payload).toBe('123') - }) + const allLower1 = vi.fn() + await pg.listen('postgresdefaultlower', allLower1) + await pg.exec(`NOTIFY postgresdefaultlower, 'payload1'`) - await db.exec('LISTEN test') - await db.exec("NOTIFY test, '123'") + const autoLowerTest1 = vi.fn() + await pg.listen('PostgresDefaultLower', autoLowerTest1) + await pg.exec(`NOTIFY PostgresDefaultLower, 'payload1'`) - await new Promise((resolve) => setTimeout(resolve, 1000)) + const autoLowerTest2 = vi.fn() + await pg.listen('PostgresDefaultLower', autoLowerTest2) + await pg.exec(`NOTIFY postgresdefaultlower, 'payload1'`) + + const autoLowerTest3 = vi.fn() + await pg.listen('postgresdefaultlower', autoLowerTest3) + await pg.exec(`NOTIFY PostgresDefaultLower, 'payload1'`) + + const caseSensitive1 = vi.fn() + await pg.listen('"tesT2"', caseSensitive1) + await pg.exec(`NOTIFY "tesT2", 'paYloAd2'`) + + const caseSensitive2 = vi.fn() + await pg.listen('"tesT3"', caseSensitive2) + await pg.exec(`NOTIFY tesT3, 'paYloAd2'`) + + const caseSensitive3 = vi.fn() + await pg.listen('testNotCalled2', caseSensitive3) + await pg.exec(`NOTIFY "testNotCalled2", 'paYloAd2'`) + + const quotedWithSpaces = vi.fn() + await pg.listen('"Quoted Channel With Spaces"', quotedWithSpaces) + await pg.exec(`NOTIFY "Quoted Channel With Spaces", 'payload1'`) + + const unquotedWithSpaces = vi.fn() + await expectToThrowAsync(async () => { + await pg.listen('Unquoted Channel With Spaces', unquotedWithSpaces) + }) + await expectToThrowAsync(async () => { + await pg.exec(`NOTIFY Unquoted Channel With Spaces, 'payload1'`) }) - it('check notify case sensitivity + special chars as Postgresql', async () => { - const pg = new PGlite({ - defaultDataTransferContainer, - }) + const otherCharsWithQuotes = vi.fn() + await pg.listen('"test&me"', otherCharsWithQuotes) + await pg.exec(`NOTIFY "test&me", 'paYloAd2'`) - const allLower1 = vi.fn() - await pg.listen('postgresdefaultlower', allLower1) - await pg.exec(`NOTIFY postgresdefaultlower, 'payload1'`) + const otherChars = vi.fn() + await expectToThrowAsync(async () => { + await pg.listen('test&me', otherChars) + }) + await expectToThrowAsync(async () => { + await pg.exec(`NOTIFY test&me, 'payload1'`) + }) - const autoLowerTest1 = vi.fn() - await pg.listen('PostgresDefaultLower', autoLowerTest1) - await pg.exec(`NOTIFY PostgresDefaultLower, 'payload1'`) + expect(allLower1).toHaveBeenCalledTimes(4) + expect(autoLowerTest1).toHaveBeenCalledTimes(3) + expect(autoLowerTest2).toHaveBeenCalledTimes(2) + expect(autoLowerTest3).toHaveBeenCalledTimes(1) + expect(caseSensitive1).toHaveBeenCalledOnce() + expect(caseSensitive2).not.toHaveBeenCalled() + expect(caseSensitive3).not.toHaveBeenCalled() + expect(otherCharsWithQuotes).toHaveBeenCalledOnce() + expect(quotedWithSpaces).toHaveBeenCalledOnce() + expect(unquotedWithSpaces).not.toHaveBeenCalled() + }) + + it('check unlisten case sensitivity + special chars as Postgresql', async () => { + const pg = new PGlite() - const autoLowerTest2 = vi.fn() - await pg.listen('PostgresDefaultLower', autoLowerTest2) + const allLower1 = vi.fn() + { + const unsub1 = await pg.listen('postgresdefaultlower', allLower1) await pg.exec(`NOTIFY postgresdefaultlower, 'payload1'`) + await unsub1() + } - const autoLowerTest3 = vi.fn() - await pg.listen('postgresdefaultlower', autoLowerTest3) + const autoLowerTest1 = vi.fn() + { + const unsub2 = await pg.listen('PostgresDefaultLower', autoLowerTest1) await pg.exec(`NOTIFY PostgresDefaultLower, 'payload1'`) + await unsub2() + } - const caseSensitive1 = vi.fn() - await pg.listen('"tesT2"', caseSensitive1) - await pg.exec(`NOTIFY "tesT2", 'paYloAd2'`) - - const caseSensitive2 = vi.fn() - await pg.listen('"tesT3"', caseSensitive2) - await pg.exec(`NOTIFY tesT3, 'paYloAd2'`) - - const caseSensitive3 = vi.fn() - await pg.listen('testNotCalled2', caseSensitive3) - await pg.exec(`NOTIFY "testNotCalled2", 'paYloAd2'`) + const autoLowerTest2 = vi.fn() + { + const unsub3 = await pg.listen('PostgresDefaultLower', autoLowerTest2) + await pg.exec(`NOTIFY postgresdefaultlower, 'payload1'`) + await unsub3() + } - const quotedWithSpaces = vi.fn() + const autoLowerTest3 = vi.fn() + { + const unsub4 = await pg.listen('postgresdefaultlower', autoLowerTest3) + await pg.exec(`NOTIFY PostgresDefaultLower, 'payload1'`) + await unsub4() + } + + const caseSensitive1 = vi.fn() + { + await pg.listen('"CaSESEnsiTIvE"', caseSensitive1) + await pg.exec(`NOTIFY "CaSESEnsiTIvE", 'payload1'`) + await pg.unlisten('"CaSESEnsiTIvE"') + await pg.exec(`NOTIFY "CaSESEnsiTIvE", 'payload1'`) + } + + const quotedWithSpaces = vi.fn() + { await pg.listen('"Quoted Channel With Spaces"', quotedWithSpaces) await pg.exec(`NOTIFY "Quoted Channel With Spaces", 'payload1'`) + await pg.unlisten('"Quoted Channel With Spaces"') + } - const unquotedWithSpaces = vi.fn() - await expectToThrowAsync(async () => { - await pg.listen('Unquoted Channel With Spaces', unquotedWithSpaces) - }) - await expectToThrowAsync(async () => { - await pg.exec(`NOTIFY Unquoted Channel With Spaces, 'payload1'`) - }) - - const otherCharsWithQuotes = vi.fn() + const otherCharsWithQuotes = vi.fn() + { await pg.listen('"test&me"', otherCharsWithQuotes) - await pg.exec(`NOTIFY "test&me", 'paYloAd2'`) - - const otherChars = vi.fn() - await expectToThrowAsync(async () => { - await pg.listen('test&me', otherChars) - }) - await expectToThrowAsync(async () => { - await pg.exec(`NOTIFY test&me, 'payload1'`) - }) - - expect(allLower1).toHaveBeenCalledTimes(4) - expect(autoLowerTest1).toHaveBeenCalledTimes(3) - expect(autoLowerTest2).toHaveBeenCalledTimes(2) - expect(autoLowerTest3).toHaveBeenCalledTimes(1) - expect(caseSensitive1).toHaveBeenCalledOnce() - expect(caseSensitive2).not.toHaveBeenCalled() - expect(caseSensitive3).not.toHaveBeenCalled() - expect(otherCharsWithQuotes).toHaveBeenCalledOnce() - expect(quotedWithSpaces).toHaveBeenCalledOnce() - expect(unquotedWithSpaces).not.toHaveBeenCalled() - }) - - it('check unlisten case sensitivity + special chars as Postgresql', async () => { - const pg = new PGlite({ - defaultDataTransferContainer, - }) - - const allLower1 = vi.fn() - { - const unsub1 = await pg.listen('postgresdefaultlower', allLower1) - await pg.exec(`NOTIFY postgresdefaultlower, 'payload1'`) - await unsub1() - } - - const autoLowerTest1 = vi.fn() - { - const unsub2 = await pg.listen('PostgresDefaultLower', autoLowerTest1) - await pg.exec(`NOTIFY PostgresDefaultLower, 'payload1'`) - await unsub2() - } - - const autoLowerTest2 = vi.fn() - { - const unsub3 = await pg.listen('PostgresDefaultLower', autoLowerTest2) - await pg.exec(`NOTIFY postgresdefaultlower, 'payload1'`) - await unsub3() - } - - const autoLowerTest3 = vi.fn() - { - const unsub4 = await pg.listen('postgresdefaultlower', autoLowerTest3) - await pg.exec(`NOTIFY PostgresDefaultLower, 'payload1'`) - await unsub4() - } - - const caseSensitive1 = vi.fn() - { - await pg.listen('"CaSESEnsiTIvE"', caseSensitive1) - await pg.exec(`NOTIFY "CaSESEnsiTIvE", 'payload1'`) - await pg.unlisten('"CaSESEnsiTIvE"') - await pg.exec(`NOTIFY "CaSESEnsiTIvE", 'payload1'`) - } - - const quotedWithSpaces = vi.fn() - { - await pg.listen('"Quoted Channel With Spaces"', quotedWithSpaces) - await pg.exec(`NOTIFY "Quoted Channel With Spaces", 'payload1'`) - await pg.unlisten('"Quoted Channel With Spaces"') - } - - const otherCharsWithQuotes = vi.fn() - { - await pg.listen('"test&me"', otherCharsWithQuotes) - await pg.exec(`NOTIFY "test&me", 'payload'`) - await pg.unlisten('"test&me"') - } - - expect(allLower1).toHaveBeenCalledOnce() - expect(autoLowerTest1).toHaveBeenCalledOnce() - expect(autoLowerTest2).toHaveBeenCalledOnce() - expect(autoLowerTest3).toHaveBeenCalledOnce() - expect(caseSensitive1).toHaveBeenCalledOnce() - expect(otherCharsWithQuotes).toHaveBeenCalledOnce() - }) + await pg.exec(`NOTIFY "test&me", 'payload'`) + await pg.unlisten('"test&me"') + } + + expect(allLower1).toHaveBeenCalledOnce() + expect(autoLowerTest1).toHaveBeenCalledOnce() + expect(autoLowerTest2).toHaveBeenCalledOnce() + expect(autoLowerTest3).toHaveBeenCalledOnce() + expect(caseSensitive1).toHaveBeenCalledOnce() + expect(otherCharsWithQuotes).toHaveBeenCalledOnce() }) }) diff --git a/packages/pglite/tests/pg_ivm.test.ts b/packages/pglite/tests/pg_ivm.test.ts index 842bc81a0..60c7aad57 100644 --- a/packages/pglite/tests/pg_ivm.test.ts +++ b/packages/pglite/tests/pg_ivm.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from 'vitest' import { testEsmCjsAndDTC } from './test-utils.ts' -await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { +await testEsmCjsAndDTC(async (importType) => { const { PGlite } = importType === 'esm' ? await import('../dist/index.js') @@ -22,7 +22,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { extensions: { pg_ivm, }, - defaultDataTransferContainer, }) await pg.exec('CREATE EXTENSION IF NOT EXISTS pg_ivm;') @@ -43,7 +42,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { extensions: { pg_ivm, }, - defaultDataTransferContainer, }) await pg.exec('CREATE EXTENSION IF NOT EXISTS pg_ivm;') @@ -89,7 +87,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { extensions: { pg_ivm, }, - defaultDataTransferContainer, }) await pg.exec('CREATE EXTENSION IF NOT EXISTS pg_ivm;') @@ -215,7 +212,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { extensions: { pg_ivm, }, - defaultDataTransferContainer, }) await pg.exec('CREATE EXTENSION IF NOT EXISTS pg_ivm;') @@ -309,7 +305,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { extensions: { pg_ivm, }, - defaultDataTransferContainer, }) await pg.exec('CREATE EXTENSION IF NOT EXISTS pg_ivm;') @@ -386,7 +381,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { extensions: { pg_ivm, }, - defaultDataTransferContainer, }) await pg.exec('CREATE EXTENSION IF NOT EXISTS pg_ivm;') diff --git a/packages/pglite/tests/pgvector.test.ts b/packages/pglite/tests/pgvector.test.ts index e27bf5d62..28f2dd9e7 100644 --- a/packages/pglite/tests/pgvector.test.ts +++ b/packages/pglite/tests/pgvector.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from 'vitest' import { testEsmCjsAndDTC } from './test-utils.ts' -await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { +await testEsmCjsAndDTC(async (importType) => { const { PGlite } = importType === 'esm' ? await import('../dist/index.js') @@ -22,7 +22,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { extensions: { vector, }, - defaultDataTransferContainer, }) await pg.exec('CREATE EXTENSION IF NOT EXISTS vector;') @@ -88,7 +87,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { extensions: { vector, }, - defaultDataTransferContainer, }) await pg.exec('CREATE EXTENSION IF NOT EXISTS vector;') @@ -114,7 +112,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { extensions: { vector, }, - defaultDataTransferContainer, }) await pg.close() @@ -124,7 +121,6 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { extensions: { vector, }, - defaultDataTransferContainer, }) await pg.exec('CREATE EXTENSION IF NOT EXISTS vector;') diff --git a/packages/pglite/tests/query-sizes.test.ts b/packages/pglite/tests/query-sizes.test.ts index cf1734824..988080b95 100644 --- a/packages/pglite/tests/query-sizes.test.ts +++ b/packages/pglite/tests/query-sizes.test.ts @@ -1,5 +1,4 @@ import { describe, it, expect, beforeEach } from 'vitest' -import { testDTC } from './test-utils.js' import { PGlite } from '../dist/index.js' function createStringOfSize(sizeInBytes: number): string { @@ -70,115 +69,113 @@ function testRowCountAndSize( } } -testDTC(async (defaultDataTransferContainer) => { - describe('query and exec with different data sizes', () => { - let db: PGlite +describe('query and exec with different data sizes', () => { + let db: PGlite - beforeEach(async () => { - db = new PGlite({ defaultDataTransferContainer, debug: 0 }) + beforeEach(async () => { + db = new PGlite({ debug: 0 }) - await db.exec(` + await db.exec(` CREATE TABLE IF NOT EXISTS size_test ( id SERIAL PRIMARY KEY, data TEXT ); `) - }) + }) - describe('exec method', () => { - testEachSize(async (_, sizeInBytes) => { - const testData = createStringOfSize(sizeInBytes) + describe('exec method', () => { + testEachSize(async (_, sizeInBytes) => { + const testData = createStringOfSize(sizeInBytes) - const results = await db.exec(` + const results = await db.exec(` INSERT INTO size_test (data) VALUES ('${testData}'); SELECT * FROM size_test; `) - expect(results).toHaveLength(2) - expect(results[1].rows).toHaveLength(1) - expect(results[1].rows[0].data).toBe(testData) - expect(results[1].rows[0].data.length).toBe(sizeInBytes) - }) + expect(results).toHaveLength(2) + expect(results[1].rows).toHaveLength(1) + expect(results[1].rows[0].data).toBe(testData) + expect(results[1].rows[0].data.length).toBe(sizeInBytes) }) + }) - describe('query method without params', () => { - testEachSize(async (_, sizeInBytes) => { - const testData = createStringOfSize(sizeInBytes) + describe('query method without params', () => { + testEachSize(async (_, sizeInBytes) => { + const testData = createStringOfSize(sizeInBytes) - await db.query(`INSERT INTO size_test (data) VALUES ('${testData}');`) + await db.query(`INSERT INTO size_test (data) VALUES ('${testData}');`) - const result = await db.query<{ id: number; data: string }>( - 'SELECT * FROM size_test;', - ) + const result = await db.query<{ id: number; data: string }>( + 'SELECT * FROM size_test;', + ) - expect(result.rows).toHaveLength(1) - expect(result.rows[0].data).toBe(testData) - expect(result.rows[0].data.length).toBe(sizeInBytes) - }) + expect(result.rows).toHaveLength(1) + expect(result.rows[0].data).toBe(testData) + expect(result.rows[0].data.length).toBe(sizeInBytes) }) + }) - describe('query method with params', () => { - testEachSize(async (_, sizeInBytes) => { - const testData = createStringOfSize(sizeInBytes) + describe('query method with params', () => { + testEachSize(async (_, sizeInBytes) => { + const testData = createStringOfSize(sizeInBytes) - await db.query('INSERT INTO size_test (data) VALUES ($1);', [testData]) + await db.query('INSERT INTO size_test (data) VALUES ($1);', [testData]) - const result = await db.query<{ id: number; data: string }>( - 'SELECT * FROM size_test WHERE data = $1;', - [testData], - ) + const result = await db.query<{ id: number; data: string }>( + 'SELECT * FROM size_test WHERE data = $1;', + [testData], + ) - expect(result.rows).toHaveLength(1) - expect(result.rows[0].data).toBe(testData) - expect(result.rows[0].data.length).toBe(sizeInBytes) - }) + expect(result.rows).toHaveLength(1) + expect(result.rows[0].data).toBe(testData) + expect(result.rows[0].data.length).toBe(sizeInBytes) }) }) +}) - describe('query with combinations of row counts and data sizes', () => { - let db: PGlite +describe('query with combinations of row counts and data sizes', () => { + let db: PGlite - beforeEach(async () => { - db = new PGlite({ defaultDataTransferContainer }) - }) + beforeEach(async () => { + db = new PGlite() + }) - testRowCountAndSize(async (_, rowCount, __, dataSize) => { - const testData = createStringOfSize(dataSize) + testRowCountAndSize(async (_, rowCount, __, dataSize) => { + const testData = createStringOfSize(dataSize) - const result = await db.query<{ id: number; data: string }>(` + const result = await db.query<{ id: number; data: string }>(` SELECT generate_series(1, ${rowCount}) as id, '${testData}' as data; `) - expect(result.rows).toHaveLength(rowCount) + expect(result.rows).toHaveLength(rowCount) - expect(result.rows[0].data).toBe(testData) - expect(result.rows[0].data.length).toBe(dataSize) - expect(result.rows[rowCount - 1].data).toBe(testData) - expect(result.rows[rowCount - 1].data.length).toBe(dataSize) - - if (rowCount > 5) { - const middleIndex = Math.floor(rowCount / 2) - expect(result.rows[middleIndex].data).toBe(testData) - expect(result.rows[middleIndex].data.length).toBe(dataSize) - } - }) + expect(result.rows[0].data).toBe(testData) + expect(result.rows[0].data.length).toBe(dataSize) + expect(result.rows[rowCount - 1].data).toBe(testData) + expect(result.rows[rowCount - 1].data.length).toBe(dataSize) + + if (rowCount > 5) { + const middleIndex = Math.floor(rowCount / 2) + expect(result.rows[middleIndex].data).toBe(testData) + expect(result.rows[middleIndex].data.length).toBe(dataSize) + } }) +}) - describe('query with postgres-generated data of different sizes', () => { - let db: PGlite +describe('query with postgres-generated data of different sizes', () => { + let db: PGlite - beforeEach(async () => { - db = new PGlite({ defaultDataTransferContainer }) - }) + beforeEach(async () => { + db = new PGlite() + }) - testEachSize(async (_, sizeInBytes) => { - const result = await db.query<{ id: number; data: string }>(` + testEachSize(async (_, sizeInBytes) => { + const result = await db.query<{ id: number; data: string }>(` SELECT 1 as id, repeat('a', ${sizeInBytes}) as data; `) - expect(result.rows).toHaveLength(1) - expect(result.rows[0].data.length).toBe(sizeInBytes) - expect(result.rows[0].data).toBe('a'.repeat(sizeInBytes)) - }) + expect(result.rows).toHaveLength(1) + expect(result.rows[0].data.length).toBe(sizeInBytes) + expect(result.rows[0].data).toBe('a'.repeat(sizeInBytes)) }) }) diff --git a/packages/pglite/tests/test-utils.ts b/packages/pglite/tests/test-utils.ts index 2318e2615..772e935cf 100644 --- a/packages/pglite/tests/test-utils.ts +++ b/packages/pglite/tests/test-utils.ts @@ -1,5 +1,4 @@ import { describe, expect } from 'vitest' -import type { DataTransferContainer } from '../dist/index.js' declare global { let Bun: any @@ -28,40 +27,16 @@ export async function expectToThrowAsync( } export async function testEsmCjsAndDTC( - fn: ( - importType: 'esm' | 'cjs', - defaultDataTransferContainer: DataTransferContainer, - ) => Promise, + fn: (importType: 'esm' | 'cjs') => Promise, ) { describe('esm import', async () => { - describe('cma data transfer container', async () => { - await fn('esm', 'cma') - }) - describe('file data transfer container', async () => { - await fn('esm', 'file') - }) + await fn('esm') }) // don't run cjs tests for Bun if (typeof Bun !== 'undefined') return describe('cjs import', async () => { - describe('cma data transfer container', async () => { - await fn('cjs', 'cma') - }) - describe('file data transfer container', async () => { - await fn('cjs', 'file') - }) - }) -} - -export async function testDTC( - fn: (defaultDataTransferContainer: DataTransferContainer) => Promise, -) { - describe('cma data transfer container', async () => { - await fn('cma') - }) - describe('file data transfer container', async () => { - await fn('file') + await fn('cjs') }) } diff --git a/packages/pglite/tests/user.test.ts b/packages/pglite/tests/user.test.ts index e9a0f9fad..23870e4da 100644 --- a/packages/pglite/tests/user.test.ts +++ b/packages/pglite/tests/user.test.ts @@ -2,21 +2,17 @@ import { describe, it, expect } from 'vitest' import { expectToThrowAsync } from './test-utils.js' import * as fs from 'fs/promises' import { PGlite } from '../dist/index.js' -import { testDTC } from './test-utils.js' -testDTC(async (defaultDataTransferContainer) => { - describe('user', () => { - it('user switching', async () => { - await fs.rm('./pgdata-test-user', { force: true, recursive: true }) +describe('user', () => { + it('user switching', async () => { + await fs.rm('./pgdata-test-user', { force: true, recursive: true }) - const db = new PGlite('./pgdata-test-user', { - defaultDataTransferContainer, - }) - await db.exec( - "CREATE USER test_user WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434';", - ) + const db = new PGlite('./pgdata-test-user') + await db.exec( + "CREATE USER test_user WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434';", + ) - await db.exec(` + await db.exec(` CREATE TABLE test ( id SERIAL PRIMARY KEY, number INT @@ -24,7 +20,7 @@ testDTC(async (defaultDataTransferContainer) => { INSERT INTO test (number) VALUES (42); `) - await db.exec(` + await db.exec(` CREATE TABLE test2 ( id SERIAL PRIMARY KEY, number INT @@ -32,48 +28,43 @@ testDTC(async (defaultDataTransferContainer) => { INSERT INTO test2 (number) VALUES (42); `) - await db.exec('ALTER TABLE test2 OWNER TO test_user;') + await db.exec('ALTER TABLE test2 OWNER TO test_user;') - await db.close() + await db.close() - const db2 = new PGlite({ - dataDir: './pgdata-test-user', - username: 'test_user', - defaultDataTransferContainer, - }) + const db2 = new PGlite({ + dataDir: './pgdata-test-user', + username: 'test_user', + }) - const currentUsername = await db2.query('SELECT current_user;') - expect(currentUsername.rows).toEqual([{ current_user: 'test_user' }]) + const currentUsername = await db2.query('SELECT current_user;') + expect(currentUsername.rows).toEqual([{ current_user: 'test_user' }]) - await expectToThrowAsync(async () => { - await db2.query('SELECT * FROM test;') - }, 'permission denied for table test') + await expectToThrowAsync(async () => { + await db2.query('SELECT * FROM test;') + }, 'permission denied for table test') - const test2 = await db2.query('SELECT * FROM test2;') - expect(test2.rows).toEqual([{ id: 1, number: 42 }]) + const test2 = await db2.query('SELECT * FROM test2;') + expect(test2.rows).toEqual([{ id: 1, number: 42 }]) - await expectToThrowAsync(async () => { - await db2.query('SET ROLE postgres;') - }, 'permission denied to set role "postgres"') - }) + await expectToThrowAsync(async () => { + await db2.query('SET ROLE postgres;') + }, 'permission denied to set role "postgres"') + }) - it('switch to user created after initial run', async () => { - await fs.rm('./pgdata-test-user', { force: true, recursive: true }) + it('switch to user created after initial run', async () => { + await fs.rm('./pgdata-test-user', { force: true, recursive: true }) - const db0 = new PGlite('./pgdata-test-user', { - defaultDataTransferContainer, - }) - await db0.waitReady - await db0.close() + const db0 = new PGlite('./pgdata-test-user') + await db0.waitReady + await db0.close() - const db = new PGlite('./pgdata-test-user', { - defaultDataTransferContainer, - }) - await db.exec( - "CREATE USER test_user WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434';", - ) + const db = new PGlite('./pgdata-test-user') + await db.exec( + "CREATE USER test_user WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434';", + ) - await db.exec(` + await db.exec(` CREATE TABLE test ( id SERIAL PRIMARY KEY, number INT @@ -81,7 +72,7 @@ testDTC(async (defaultDataTransferContainer) => { INSERT INTO test (number) VALUES (42); `) - await db.exec(` + await db.exec(` CREATE TABLE test2 ( id SERIAL PRIMARY KEY, number INT @@ -89,53 +80,50 @@ testDTC(async (defaultDataTransferContainer) => { INSERT INTO test2 (number) VALUES (42); `) - await db.exec('ALTER TABLE test2 OWNER TO test_user;') + await db.exec('ALTER TABLE test2 OWNER TO test_user;') - await db.close() + await db.close() - const db2 = new PGlite({ - dataDir: './pgdata-test-user', - username: 'test_user', - }) + const db2 = new PGlite({ + dataDir: './pgdata-test-user', + username: 'test_user', + }) - const currentUsername = await db2.query('SELECT current_user;') - expect(currentUsername.rows).toEqual([{ current_user: 'test_user' }]) + const currentUsername = await db2.query('SELECT current_user;') + expect(currentUsername.rows).toEqual([{ current_user: 'test_user' }]) - await expectToThrowAsync(async () => { - await db2.query('SELECT * FROM test;') - }, 'permission denied for table test') + await expectToThrowAsync(async () => { + await db2.query('SELECT * FROM test;') + }, 'permission denied for table test') - const test2 = await db2.query('SELECT * FROM test2;') - expect(test2.rows).toEqual([{ id: 1, number: 42 }]) + const test2 = await db2.query('SELECT * FROM test2;') + expect(test2.rows).toEqual([{ id: 1, number: 42 }]) - await expectToThrowAsync(async () => { - await db2.query('SET ROLE postgres;') - }, 'permission denied to set role "postgres"') - }) + await expectToThrowAsync(async () => { + await db2.query('SET ROLE postgres;') + }, 'permission denied to set role "postgres"') + }) + + it('create database and switch to it', async () => { + await fs.rm('./pgdata-test-user', { force: true, recursive: true }) + + const db = new PGlite('./pgdata-test-user') + await db.exec( + "CREATE USER test_user WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434';", + ) - it('create database and switch to it', async () => { - await fs.rm('./pgdata-test-user', { force: true, recursive: true }) - - const db = new PGlite('./pgdata-test-user', { - defaultDataTransferContainer, - }) - await db.exec( - "CREATE USER test_user WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434';", - ) - - await db.exec('CREATE DATABASE test_db OWNER test_user;') - await db.close() - - const db2 = new PGlite({ - dataDir: './pgdata-test-user', - username: 'test_user', - database: 'test_db', - }) - - const currentUsername = await db2.query('SELECT current_user;') - expect(currentUsername.rows).toEqual([{ current_user: 'test_user' }]) - const currentDatabase = await db2.query('SELECT current_database();') - expect(currentDatabase.rows).toEqual([{ current_database: 'test_db' }]) + await db.exec('CREATE DATABASE test_db OWNER test_user;') + await db.close() + + const db2 = new PGlite({ + dataDir: './pgdata-test-user', + username: 'test_user', + database: 'test_db', }) + + const currentUsername = await db2.query('SELECT current_user;') + expect(currentUsername.rows).toEqual([{ current_user: 'test_user' }]) + const currentDatabase = await db2.query('SELECT current_database();') + expect(currentDatabase.rows).toEqual([{ current_database: 'test_db' }]) }) }) From d2f112c2f48fb9d24f3bf2896985fdc185f1dcee Mon Sep 17 00:00:00 2001 From: tdrz Date: Wed, 18 Jun 2025 15:40:29 +0200 Subject: [PATCH 03/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index b4921ef0b..fe8fb0f01 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit b4921ef0b4a2cadc7943c90119d2acdfc6380c73 +Subproject commit fe8fb0f015945997b08558de21567a66127dd7f5 From 5c41321d6787bc847a899aa16d58eb192a70f19f Mon Sep 17 00:00:00 2001 From: tdrz Date: Tue, 8 Jul 2025 09:48:03 +0200 Subject: [PATCH 04/79] cleanup --- packages/pglite/src/pglite.ts | 78 +---------------------------------- 1 file changed, 2 insertions(+), 76 deletions(-) diff --git a/packages/pglite/src/pglite.ts b/packages/pglite/src/pglite.ts index b23f01107..c71f8c078 100644 --- a/packages/pglite/src/pglite.ts +++ b/packages/pglite/src/pglite.ts @@ -75,7 +75,7 @@ export class PGlite #notifyListeners = new Map void>>() #globalNotifyListeners = new Set<(channel: string, payload: string) => void>() - static readonly RECV_BUF_SIZE: number = 10 * 1024 * 1024 // 10MB default + static readonly RECV_BUF_SIZE: number = 16 * 1024 * 1024 // 16MB default // receive data from wasm #onWriteDataPtr: number = -1 @@ -407,10 +407,6 @@ export class PGlite this.#inputData.set(copied, this.#writeOffset) this.#writeOffset += copied.length - // const newAccumulated = new Uint8Array(this.#accumulatedData.length + copied.length); - // newAccumulated.set(this.#accumulatedData, 0); - // newAccumulated.set(copied, this.#accumulatedData.length); - // this.#accumulatedData = newAccumulated; return this.#inputData.length }, 'iii', @@ -653,40 +649,7 @@ export class PGlite const mod = this.mod! // >0 set buffer content type to wire protocol mod._use_wire(1) - // const msg_len = message.length - - // TODO: if (message.length>CMA_B) force file - - // let currDataTransferContainer = - // options.dataTransferContainer ?? this.#dataTransferContainer - - // do we overflow allocated shared memory segment - // if (message.length >= mod.FD_BUFFER_MAX) currDataTransferContainer = 'file' - - // switch (currDataTransferContainer) { - // case 'cma': { - // // set buffer size so answer will be at size+0x2 pointer addr - // mod._interactive_write(message.length) - // // TODO: make it seg num * seg maxsize if multiple channels. - // mod.HEAPU8.set(message, 1) - // break - // } - // case 'file': { - // // Use socketfiles to emulate a socket connection - // const pg_lck = '/tmp/pglite/base/.s.PGSQL.5432.lck.in' - // const pg_in = '/tmp/pglite/base/.s.PGSQL.5432.in' - // mod._interactive_write(0) - // mod.FS.writeFile(pg_lck, message) - // mod.FS.rename(pg_lck, pg_in) - // break - // } - // default: - // throw new Error( - // `Unknown data transfer container: ${currDataTransferContainer}`, - // ) - // } - - // this.#accumulatedData = [] + if (this.#inputData.buffer.byteLength > PGlite.RECV_BUF_SIZE) { this.#inputData = new Uint8Array(PGlite.RECV_BUF_SIZE) } @@ -700,43 +663,6 @@ export class PGlite this.#outputData = [] - // const channel = mod._get_channel() - // if (channel < 0) currDataTransferContainer = 'file' - - // // TODO: use channel value for msg_start - // if (channel > 0) currDataTransferContainer = 'cma' - - // switch (currDataTransferContainer) { - // case 'cma': { - // // Read responses from the buffer - - // const msg_start = msg_len + 2 - // const msg_end = msg_start + mod._interactive_read() - // data = mod.HEAPU8.subarray(msg_start, msg_end) - // break - // } - // case 'file': { - // // Use socketfiles to emulate a socket connection - // const pg_out = '/tmp/pglite/base/.s.PGSQL.5432.out' - // try { - // const fstat = mod.FS.stat(pg_out) - // const stream = mod.FS.open(pg_out, 'r') - // data = new Uint8Array(fstat.size) - // mod.FS.read(stream, data, 0, fstat.size, 0) - // mod.FS.unlink(pg_out) - // } catch (x) { - // // case of single X message. - // data = new Uint8Array(0) - // } - // break - // } - // default: - // throw new Error( - // `Unknown data transfer container: ${currDataTransferContainer}`, - // ) - // } - - // return data if (this.#writeOffset) return this.#inputData.subarray(0, this.#writeOffset) return new Uint8Array(0) } From f6c512cdf6ab4ab68babb74ab94e56f96edcc7bc Mon Sep 17 00:00:00 2001 From: tdrz Date: Tue, 8 Jul 2025 15:51:06 +0200 Subject: [PATCH 05/79] style --- packages/pglite/src/pglite.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pglite/src/pglite.ts b/packages/pglite/src/pglite.ts index c71f8c078..39106905d 100644 --- a/packages/pglite/src/pglite.ts +++ b/packages/pglite/src/pglite.ts @@ -649,7 +649,7 @@ export class PGlite const mod = this.mod! // >0 set buffer content type to wire protocol mod._use_wire(1) - + if (this.#inputData.buffer.byteLength > PGlite.RECV_BUF_SIZE) { this.#inputData = new Uint8Array(PGlite.RECV_BUF_SIZE) } From 34eff391d525c246a7efd5d05ceacb3e391f4243 Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 15 Jul 2025 15:49:10 +0200 Subject: [PATCH 06/79] change buffer size expansion strategy --- packages/pglite/src/pglite.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/pglite/src/pglite.ts b/packages/pglite/src/pglite.ts index 39106905d..ee5fd3177 100644 --- a/packages/pglite/src/pglite.ts +++ b/packages/pglite/src/pglite.ts @@ -393,12 +393,10 @@ export class PGlite const requiredSize = this.#writeOffset + copied.length if (requiredSize > this.#inputData.length) { - // Expand buffer size (double until it fits) - let newSize = this.#inputData.length - while (newSize < requiredSize) { - newSize *= 2 - } - + const newSize = + this.#inputData.length + + (this.#inputData.length >> 1) + + requiredSize const newBuffer = new Uint8Array(newSize) newBuffer.set(this.#inputData.subarray(0, this.#writeOffset)) this.#inputData = newBuffer From da19cf0a4aef9aa41dbcd6562c136c5e8e1d0153 Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 23 Sep 2025 16:59:53 +0200 Subject: [PATCH 07/79] fixes #657 --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index c0269765e..9bea834b4 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit c0269765ed89b8df428a91192d16ffbfdb8176e0 +Subproject commit 9bea834b44d63ae7925a966e4ebbd1aeebd8cac1 From 30a139721e9046746f479757ccb5f5d95c25a725 Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 23 Sep 2025 17:03:33 +0200 Subject: [PATCH 08/79] changeset --- .changeset/slimy-countries-rush.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/slimy-countries-rush.md diff --git a/.changeset/slimy-countries-rush.md b/.changeset/slimy-countries-rush.md new file mode 100644 index 000000000..e87d93f90 --- /dev/null +++ b/.changeset/slimy-countries-rush.md @@ -0,0 +1,5 @@ +--- +'@electric-sql/pglite': patch +--- + +initdb calls system to query the server configs. avoid that by hardcoding a return value of 123 From b607f50ebf0880f9bba3ee6e9fae32dce1212f85 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 11:19:22 +0200 Subject: [PATCH 09/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 9bea834b4..52b61d867 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 9bea834b44d63ae7925a966e4ebbd1aeebd8cac1 +Subproject commit 52b61d867ff4626cc29b09ce3ecbc0b3074ee028 From e2583758b1268156f1740f0968e61a75f91b1633 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 13:32:04 +0200 Subject: [PATCH 10/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 52b61d867..7e4257c03 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 52b61d867ff4626cc29b09ce3ecbc0b3074ee028 +Subproject commit 7e4257c0366b8e97391773eaa87c4aca1bb5c6ef From 61af32abb3d1f5bde71580de1ccbfef66693e941 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 13:49:39 +0200 Subject: [PATCH 11/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 7e4257c03..09a566660 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 7e4257c0366b8e97391773eaa87c4aca1bb5c6ef +Subproject commit 09a56666044322b3b29b1e294cbf5c33552d1b9f From b8dba43bf291d599f095f08c652894f5dc8f8c89 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 14:19:10 +0200 Subject: [PATCH 12/79] remove get_buffer_size and get_buffer_addr" --- packages/pglite/src/postgresMod.ts | 2 -- postgres-pglite | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/pglite/src/postgresMod.ts b/packages/pglite/src/postgresMod.ts index abb4a0155..258f1131e 100644 --- a/packages/pglite/src/postgresMod.ts +++ b/packages/pglite/src/postgresMod.ts @@ -28,8 +28,6 @@ export interface PostgresMod _pgl_initdb: () => number _pgl_backend: () => void _pgl_shutdown: () => void - _get_buffer_size: (fd: number) => number - _get_buffer_addr: (fd: number) => number _get_channel: () => number _interactive_write: (msgLength: number) => void _interactive_one: () => void diff --git a/postgres-pglite b/postgres-pglite index 09a566660..c57040479 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 09a56666044322b3b29b1e294cbf5c33552d1b9f +Subproject commit c57040479cbd88007e41be22431c97a9b51a7012 From e9047f2970009b732de1e9a048a758e7d734dfc3 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 14:42:46 +0200 Subject: [PATCH 13/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index c57040479..8f1bad613 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit c57040479cbd88007e41be22431c97a9b51a7012 +Subproject commit 8f1bad613caee84196e32d78d86f7ae94ea24fc4 From 4598daa7317fea7e81ffaa349b17754d17e22383 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 15:10:19 +0200 Subject: [PATCH 14/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 8f1bad613..bb1befd6e 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 8f1bad613caee84196e32d78d86f7ae94ea24fc4 +Subproject commit bb1befd6e74e12327a85c1eabe91b26e61366dba From 5494735bc2ce3a143e21373a764b5bd73ff7e7af Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 17:41:33 +0200 Subject: [PATCH 15/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index bb1befd6e..7c73e9c21 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit bb1befd6e74e12327a85c1eabe91b26e61366dba +Subproject commit 7c73e9c21660fec05ee534368d41d0b98df83306 From ef11f15cb0dfff683357aaad59320ef8627488ef Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 17:49:20 +0200 Subject: [PATCH 16/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 7c73e9c21..58724c569 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 7c73e9c21660fec05ee534368d41d0b98df83306 +Subproject commit 58724c5692e33d08e73ea3df4d8bda1d2d2b6d6e From 1c917adee5fd265afac54663c04f1898287ed1b6 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 18:03:17 +0200 Subject: [PATCH 17/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 58724c569..b4b9af3a5 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 58724c5692e33d08e73ea3df4d8bda1d2d2b6d6e +Subproject commit b4b9af3a5514d39096c1aa26237a2ef71912e373 From 6d9a9639270cae2a782245b9ccd4ae323ffba53f Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 19:38:19 +0200 Subject: [PATCH 18/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index b4b9af3a5..433ce1eb8 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit b4b9af3a5514d39096c1aa26237a2ef71912e373 +Subproject commit 433ce1eb8729011b777bf467a2170927e9cd4426 From c52ae191f0b8a4ac89d0d38c2a046e1f68d0df8e Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 19:40:38 +0200 Subject: [PATCH 19/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 433ce1eb8..4b7eafe64 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 433ce1eb8729011b777bf467a2170927e9cd4426 +Subproject commit 4b7eafe64580fd8d409c2dfcb0104a4254dfecad From 61d9dbaec31ace5939efc9ff583fce90b5751dca Mon Sep 17 00:00:00 2001 From: tudor Date: Sun, 28 Sep 2025 18:33:34 +0200 Subject: [PATCH 20/79] try again --- packages/pglite/package.json | 10 +++ packages/pglite/src/postgis/index.ts | 17 ++++++ packages/pglite/tests/postgis.test.ts | 87 +++++++++++++++++++++++++++ packages/pglite/tsup.config.ts | 1 + postgres-pglite | 2 +- 5 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 packages/pglite/src/postgis/index.ts create mode 100644 packages/pglite/tests/postgis.test.ts diff --git a/packages/pglite/package.json b/packages/pglite/package.json index de0c9bcd5..cd7197a04 100644 --- a/packages/pglite/package.json +++ b/packages/pglite/package.json @@ -80,6 +80,16 @@ "default": "./dist/pg_ivm/index.cjs" } }, + "./postgis": { + "import": { + "types": "./dist/postgis/index.d.ts", + "default": "./dist/postgis/index.js" + }, + "require": { + "types": "./dist/postgis/index.d.cts", + "default": "./dist/postgis/index.cjs" + } + }, "./nodefs": { "import": { "types": "./dist/fs/nodefs.d.ts", diff --git a/packages/pglite/src/postgis/index.ts b/packages/pglite/src/postgis/index.ts new file mode 100644 index 000000000..9e4e37d95 --- /dev/null +++ b/packages/pglite/src/postgis/index.ts @@ -0,0 +1,17 @@ +import type { + Extension, + ExtensionSetupResult, + PGliteInterface, +} from '../interface' + +const setup = async (_pg: PGliteInterface, emscriptenOpts: any) => { + return { + emscriptenOpts, + bundlePath: new URL('../../release/postgis.tar.gz', import.meta.url), + } satisfies ExtensionSetupResult +} + +export const postgis = { + name: 'postgis', + setup, +} satisfies Extension diff --git a/packages/pglite/tests/postgis.test.ts b/packages/pglite/tests/postgis.test.ts new file mode 100644 index 000000000..e56619e7d --- /dev/null +++ b/packages/pglite/tests/postgis.test.ts @@ -0,0 +1,87 @@ +import { describe, it, expect } from 'vitest' +import { testEsmCjsAndDTC } from './test-utils.ts' + +await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { + const { PGlite } = + importType === 'esm' + ? await import('../dist/index.js') + : ((await import( + '../dist/index.cjs' + )) as unknown as typeof import('../dist/index.js')) + + const { postgis } = + importType === 'esm' + ? await import('../dist/postgis/index.js') + : ((await import( + '../dist/postgis/index.cjs' + )) as unknown as typeof import('../dist/postgis/index.js')) + + describe(`postgis`, () => { + it('basic', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + defaultDataTransferContainer, + }) + + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + await pg.exec(` + CREATE TABLE vehicle_location ( + time TIMESTAMPTZ NOT NULL, + vehicle_id INT NOT NULL, + location GEOGRAPHY(POINT, 4326) +); + `) + await pg.exec(`INSERT INTO vehicle_location VALUES + ('2023-05-29 20:00:00', 1, 'POINT(15.3672 -87.7231)'), + ('2023-05-30 20:00:00', 1, 'POINT(15.3652 -80.7331)'), + ('2023-05-31 20:00:00', 1, 'POINT(15.2672 -85.7431)');`) + +// const res = await pg.exec(` +// SELECT +// name, +// vec, +// vec <-> '[3,1,2]' AS distance +// FROM test; +// `) + +// expect(res).toMatchObject([ +// { +// rows: [ +// { +// name: 'test1', +// vec: '[1,2,3]', +// distance: 2.449489742783178, +// }, +// { +// name: 'test2', +// vec: '[4,5,6]', +// distance: 5.744562646538029, +// }, +// { +// name: 'test3', +// vec: '[7,8,9]', +// distance: 10.677078252031311, +// }, +// ], +// fields: [ +// { +// name: 'name', +// dataTypeID: 25, +// }, +// { +// name: 'vec', +// dataTypeID: 16385, +// }, +// { +// name: 'distance', +// dataTypeID: 701, +// }, +// ], +// affectedRows: 0, +// }, +// ]) + }) + }) +}) diff --git a/packages/pglite/tsup.config.ts b/packages/pglite/tsup.config.ts index 65d6508c3..81e2c183a 100644 --- a/packages/pglite/tsup.config.ts +++ b/packages/pglite/tsup.config.ts @@ -25,6 +25,7 @@ const entryPoints = [ 'src/live/index.ts', 'src/vector/index.ts', 'src/pg_ivm/index.ts', + 'src/postgis/index.ts', 'src/worker/index.ts', ] diff --git a/postgres-pglite b/postgres-pglite index 9bea834b4..4ba6f2c86 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 9bea834b44d63ae7925a966e4ebbd1aeebd8cac1 +Subproject commit 4ba6f2c86b5557eca6d6ce18e76f5fdb3176e3ac From d6a55a18f138d3a76d6c50850dad66f6b0822c2a Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 29 Sep 2025 10:32:56 +0200 Subject: [PATCH 21/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 4ba6f2c86..1fb2e61c3 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 4ba6f2c86b5557eca6d6ce18e76f5fdb3176e3ac +Subproject commit 1fb2e61c34a6ce8703cc12ca07a212cefecc0359 From 415e62cbfe45eb26742170da418dfce54ed70db5 Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 29 Sep 2025 15:43:35 +0200 Subject: [PATCH 22/79] submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 1fb2e61c3..7a7021790 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 1fb2e61c34a6ce8703cc12ca07a212cefecc0359 +Subproject commit 7a7021790922e71b3764e80dc4b4e8795f6f4e7d From a962996088d65450d140bb0e083c1e9a3a21084e Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 29 Sep 2025 16:29:42 +0200 Subject: [PATCH 23/79] submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 7a7021790..fb80ac4c6 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 7a7021790922e71b3764e80dc4b4e8795f6f4e7d +Subproject commit fb80ac4c6e37092b556654a9f59cf71831d7ad98 From c9e7ddad823a263a2ed0ebb8b7f11dc57a3482a8 Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 30 Sep 2025 14:35:11 +0200 Subject: [PATCH 24/79] more tests; updated submodules --- packages/pglite/tests/postgis.test.ts | 75 ++++++++++++--------------- postgres-pglite | 2 +- 2 files changed, 33 insertions(+), 44 deletions(-) diff --git a/packages/pglite/tests/postgis.test.ts b/packages/pglite/tests/postgis.test.ts index e56619e7d..f33ec14d5 100644 --- a/packages/pglite/tests/postgis.test.ts +++ b/packages/pglite/tests/postgis.test.ts @@ -38,50 +38,39 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { ('2023-05-30 20:00:00', 1, 'POINT(15.3652 -80.7331)'), ('2023-05-31 20:00:00', 1, 'POINT(15.2672 -85.7431)');`) -// const res = await pg.exec(` -// SELECT -// name, -// vec, -// vec <-> '[3,1,2]' AS distance -// FROM test; -// `) -// expect(res).toMatchObject([ -// { -// rows: [ -// { -// name: 'test1', -// vec: '[1,2,3]', -// distance: 2.449489742783178, -// }, -// { -// name: 'test2', -// vec: '[4,5,6]', -// distance: 5.744562646538029, -// }, -// { -// name: 'test3', -// vec: '[7,8,9]', -// distance: 10.677078252031311, -// }, -// ], -// fields: [ -// { -// name: 'name', -// dataTypeID: 25, -// }, -// { -// name: 'vec', -// dataTypeID: 16385, -// }, -// { -// name: 'distance', -// dataTypeID: 701, -// }, -// ], -// affectedRows: 0, -// }, -// ]) + }), + it('cities', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + defaultDataTransferContainer, + }) + + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + await pg.exec(` + CREATE TABLE cities ( + id SERIAL PRIMARY KEY, + name VARCHAR(100), + location GEOMETRY(Point, 4326) +); + `) + await pg.exec(`INSERT INTO cities (name, location) +VALUES + ('New York', ST_GeomFromText('POINT(-74.0060 40.7128)', 4326)), + ('Los Angeles', ST_GeomFromText('POINT(-118.2437 34.0522)', 4326)), + ('Chicago', ST_GeomFromText('POINT(-87.6298 41.8781)', 4326));`) + + await pg.exec(`WITH state_boundary AS ( + SELECT ST_GeomFromText( + 'POLYGON((-91 36, -91 43, -87 43, -87 36, -91 36))', 4326 + ) AS geom +) +SELECT c.name +FROM cities c, state_boundary s +WHERE ST_Within(c.location, s.geom);`) + }) }) }) diff --git a/postgres-pglite b/postgres-pglite index fb80ac4c6..1e4f3c000 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit fb80ac4c6e37092b556654a9f59cf71831d7ad98 +Subproject commit 1e4f3c0002ace80f5d75ee5789d3249ea65e4064 From 67535333474ef6181aa0772800f98f7feada7509 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 1 Oct 2025 21:55:46 +0200 Subject: [PATCH 25/79] submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 4b7eafe64..6cf2d5fea 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 4b7eafe64580fd8d409c2dfcb0104a4254dfecad +Subproject commit 6cf2d5fea350853045d1c50c274da689693c63d9 From d4f4cbbc947bafb1041015fd62234d99bff31dfd Mon Sep 17 00:00:00 2001 From: tudor Date: Thu, 2 Oct 2025 11:55:40 +0200 Subject: [PATCH 26/79] add final result messages --- packages/pglite/src/base.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pglite/src/base.ts b/packages/pglite/src/base.ts index 3817c8a12..2563198ae 100644 --- a/packages/pglite/src/base.ts +++ b/packages/pglite/src/base.ts @@ -228,7 +228,7 @@ export abstract class BasePGlite this.#log('runQuery', query, params, options) await this._handleBlob(options?.blob) - let results + let results = [] try { const { messages: parseResults } = await this.#execProtocolNoSync( @@ -288,7 +288,7 @@ export abstract class BasePGlite } throw e } finally { - await this.#execProtocolNoSync(serializeProtocol.sync(), options) + results.push(...(await this.#execProtocolNoSync(serializeProtocol.sync(), options)).messages) } await this._cleanupBlob() From 5563e0e6a62291bc6300e4f74ea18129d5234991 Mon Sep 17 00:00:00 2001 From: tudor Date: Thu, 2 Oct 2025 11:55:47 +0200 Subject: [PATCH 27/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 6cf2d5fea..fd02949cd 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 6cf2d5fea350853045d1c50c274da689693c63d9 +Subproject commit fd02949cd5ac26ecfb0a15ef8f78bb54ca803cb4 From 7b075961c29f58a97e6ffc9b10390326c1e3014f Mon Sep 17 00:00:00 2001 From: tudor Date: Thu, 2 Oct 2025 12:05:31 +0200 Subject: [PATCH 28/79] renaming; prettify --- packages/pglite/src/base.ts | 5 ++++- packages/pglite/src/pglite.ts | 10 +++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/pglite/src/base.ts b/packages/pglite/src/base.ts index 2563198ae..f5335d983 100644 --- a/packages/pglite/src/base.ts +++ b/packages/pglite/src/base.ts @@ -288,7 +288,10 @@ export abstract class BasePGlite } throw e } finally { - results.push(...(await this.#execProtocolNoSync(serializeProtocol.sync(), options)).messages) + results.push( + ...(await this.#execProtocolNoSync(serializeProtocol.sync(), options)) + .messages, + ) } await this._cleanupBlob() diff --git a/packages/pglite/src/pglite.ts b/packages/pglite/src/pglite.ts index a9f8c232e..387c04bd6 100644 --- a/packages/pglite/src/pglite.ts +++ b/packages/pglite/src/pglite.ts @@ -78,14 +78,14 @@ export class PGlite static readonly RECV_BUF_SIZE: number = 16 * 1024 * 1024 // 16MB default // receive data from wasm - #onWriteDataPtr: number = -1 + #pglite_write: number = -1 // buffer that holds data received from wasm #inputData = new Uint8Array(PGlite.RECV_BUF_SIZE) // write index in the buffer #writeOffset: number = 0 // send data to wasm - #onReadDataPtr: number = -1 + #pglite_read: number = -1 // buffer that holds the data to be sent to wasm #outputData: any = [] // read index in the buffer @@ -379,7 +379,7 @@ export class PGlite this.mod = await PostgresModFactory(emscriptenOpts) // set the write callback - this.#onWriteDataPtr = (this.mod as any).addFunction( + this.#pglite_write = (this.mod as any).addFunction( (ptr: any, length: number) => { let bytes try { @@ -411,7 +411,7 @@ export class PGlite ) // set the read callback - this.#onReadDataPtr = (this.mod as any).addFunction( + this.#pglite_read = (this.mod as any).addFunction( (ptr: any, max_length: number) => { // copy current data to wasm buffer let length = this.#outputData.length - this.#readOffset @@ -435,7 +435,7 @@ export class PGlite 'iii', ) - this.mod._set_read_write_cbs(this.#onReadDataPtr, this.#onWriteDataPtr) + this.mod._set_read_write_cbs(this.#pglite_read, this.#pglite_write) // Sync the filesystem from any previous store await this.fs!.initialSyncFs() From 28817bcc647aec9234b221464ae250deafe94f0a Mon Sep 17 00:00:00 2001 From: tudor Date: Thu, 2 Oct 2025 12:14:10 +0200 Subject: [PATCH 29/79] test update --- packages/pglite/tests/message-context-leak.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pglite/tests/message-context-leak.test.ts b/packages/pglite/tests/message-context-leak.test.ts index 98ed93e3f..92639a0aa 100644 --- a/packages/pglite/tests/message-context-leak.test.ts +++ b/packages/pglite/tests/message-context-leak.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach } from 'vitest' -import { testDTC } from './test-utils.js' +import { testEsmCjsAndDTC } from './test-utils.js' import { PGlite } from '../dist/index.js' // This test isolates the MessageContext leak reported in @@ -16,7 +16,7 @@ function makeJsonBlob(size: number): string { return JSON.stringify({ padding: 'x'.repeat(size) }) } -testDTC(async () => { +testEsmCjsAndDTC(async () => { describe('MessageContext reset between queries', () => { let db: PGlite From 365b236fa183e42914ecfca831ab96b0459c6e72 Mon Sep 17 00:00:00 2001 From: tudor Date: Thu, 2 Oct 2025 21:36:14 +0200 Subject: [PATCH 30/79] run sync after getting types from db --- packages/pglite/src/base.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/pglite/src/base.ts b/packages/pglite/src/base.ts index f5335d983..9ad02cee9 100644 --- a/packages/pglite/src/base.ts +++ b/packages/pglite/src/base.ts @@ -237,12 +237,16 @@ export abstract class BasePGlite ) const dataTypeIDs = parseDescribeStatementResults( - ( + [...( await this.#execProtocolNoSync( serializeProtocol.describe({ type: 'S' }), options, ) ).messages, + ... + (await this.#execProtocolNoSync(serializeProtocol.sync(), options)) + .messages, + ] ) const values = params.map((param, i) => { From 261c3f8888b8b0927bd02e5d61380d0689d10a5a Mon Sep 17 00:00:00 2001 From: tudor Date: Fri, 3 Oct 2025 09:00:11 +0200 Subject: [PATCH 31/79] add results returned by sync --- packages/pglite/src/base.ts | 16 ++++++++++------ packages/pglite/src/utils.ts | 14 ++++++-------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/packages/pglite/src/base.ts b/packages/pglite/src/base.ts index 9ad02cee9..94ab64aab 100644 --- a/packages/pglite/src/base.ts +++ b/packages/pglite/src/base.ts @@ -322,7 +322,7 @@ export abstract class BasePGlite // No params so we can just send the query this.#log('runExec', query, options) await this._handleBlob(options?.blob) - let results + let results = [] try { results = ( await this.#execProtocolNoSync( @@ -342,7 +342,7 @@ export abstract class BasePGlite } throw e } finally { - await this.#execProtocolNoSync(serializeProtocol.sync(), options) + results.push(...(await this.#execProtocolNoSync(serializeProtocol.sync(), options)).messages) } this._cleanupBlob() if (!this.#inTransaction) { @@ -373,15 +373,19 @@ export abstract class BasePGlite options, ) - const describeResults = await this.#execProtocolNoSync( + let messages = (await this.#execProtocolNoSync( serializeProtocol.describe({ type: 'S' }), options, - ) - const paramDescription = describeResults.messages.find( + )).messages + + messages.push(...(await this.#execProtocolNoSync(serializeProtocol.sync(), + options)).messages) + + const paramDescription = messages.find( (msg): msg is ParameterDescriptionMessage => msg.name === 'parameterDescription', ) - const resultDescription = describeResults.messages.find( + const resultDescription = messages.find( (msg): msg is RowDescriptionMessage => msg.name === 'rowDescription', ) diff --git a/packages/pglite/src/utils.ts b/packages/pglite/src/utils.ts index 5f544f1cd..e7573c10a 100644 --- a/packages/pglite/src/utils.ts +++ b/packages/pglite/src/utils.ts @@ -142,23 +142,21 @@ export async function formatQuery( tx = tx ?? pg // Get the types of the parameters - let dataTypeIDs: number[] + let messages = [] try { await pg.execProtocol(serializeProtocol.parse({ text: query }), { syncToFs: false, }) - dataTypeIDs = parseDescribeStatementResults( - ( - await pg.execProtocol(serializeProtocol.describe({ type: 'S' }), { + messages.push(...(await pg.execProtocol(serializeProtocol.describe({ type: 'S' }), { syncToFs: false, - }) - ).messages, - ) + })).messages) } finally { - await pg.execProtocol(serializeProtocol.sync(), { syncToFs: false }) + messages.push(...(await pg.execProtocol(serializeProtocol.sync(), { syncToFs: false })).messages) } + const dataTypeIDs = parseDescribeStatementResults(messages) + // replace $1, $2, etc with %1L, %2L, etc const subbedQuery = query.replace(/\$([0-9]+)/g, (_, num) => { return '%' + num + 'L' From 2c57d38bc32c2c66b001051a056ac92e58786eb7 Mon Sep 17 00:00:00 2001 From: tudor Date: Fri, 3 Oct 2025 09:03:51 +0200 Subject: [PATCH 32/79] style --- packages/pglite/src/base.ts | 33 +++++++++++++++++++-------------- packages/pglite/src/utils.ts | 17 ++++++++++++----- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/packages/pglite/src/base.ts b/packages/pglite/src/base.ts index 94ab64aab..f06bb100a 100644 --- a/packages/pglite/src/base.ts +++ b/packages/pglite/src/base.ts @@ -236,18 +236,16 @@ export abstract class BasePGlite options, ) - const dataTypeIDs = parseDescribeStatementResults( - [...( + const dataTypeIDs = parseDescribeStatementResults([ + ...( await this.#execProtocolNoSync( serializeProtocol.describe({ type: 'S' }), options, ) ).messages, - ... - (await this.#execProtocolNoSync(serializeProtocol.sync(), options)) - .messages, - ] - ) + ...(await this.#execProtocolNoSync(serializeProtocol.sync(), options)) + .messages, + ]) const values = params.map((param, i) => { const oid = dataTypeIDs[i] @@ -342,7 +340,10 @@ export abstract class BasePGlite } throw e } finally { - results.push(...(await this.#execProtocolNoSync(serializeProtocol.sync(), options)).messages) + results.push( + ...(await this.#execProtocolNoSync(serializeProtocol.sync(), options)) + .messages, + ) } this._cleanupBlob() if (!this.#inTransaction) { @@ -373,13 +374,17 @@ export abstract class BasePGlite options, ) - let messages = (await this.#execProtocolNoSync( - serializeProtocol.describe({ type: 'S' }), - options, - )).messages + const messages = ( + await this.#execProtocolNoSync( + serializeProtocol.describe({ type: 'S' }), + options, + ) + ).messages - messages.push(...(await this.#execProtocolNoSync(serializeProtocol.sync(), - options)).messages) + messages.push( + ...(await this.#execProtocolNoSync(serializeProtocol.sync(), options)) + .messages, + ) const paramDescription = messages.find( (msg): msg is ParameterDescriptionMessage => diff --git a/packages/pglite/src/utils.ts b/packages/pglite/src/utils.ts index e7573c10a..0cbde2f34 100644 --- a/packages/pglite/src/utils.ts +++ b/packages/pglite/src/utils.ts @@ -142,20 +142,27 @@ export async function formatQuery( tx = tx ?? pg // Get the types of the parameters - let messages = [] + const messages = [] try { await pg.execProtocol(serializeProtocol.parse({ text: query }), { syncToFs: false, }) - messages.push(...(await pg.execProtocol(serializeProtocol.describe({ type: 'S' }), { + messages.push( + ...( + await pg.execProtocol(serializeProtocol.describe({ type: 'S' }), { syncToFs: false, - })).messages) + }) + ).messages, + ) } finally { - messages.push(...(await pg.execProtocol(serializeProtocol.sync(), { syncToFs: false })).messages) + messages.push( + ...(await pg.execProtocol(serializeProtocol.sync(), { syncToFs: false })) + .messages, + ) } - const dataTypeIDs = parseDescribeStatementResults(messages) + const dataTypeIDs = parseDescribeStatementResults(messages) // replace $1, $2, etc with %1L, %2L, etc const subbedQuery = query.replace(/\$([0-9]+)/g, (_, num) => { From 2b97c3bb7fb66776a05cf1c623d2b2f4cd9054d7 Mon Sep 17 00:00:00 2001 From: tudor Date: Fri, 3 Oct 2025 21:53:58 +0200 Subject: [PATCH 33/79] submodule; improved build command --- package.json | 2 +- postgres-pglite | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 96a557ed8..fa32c00fd 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "ci:publish": "pnpm changeset publish", "ts:build": "pnpm -r --filter \"./packages/**\" build", "ts:build:debug": "DEBUG=true pnpm ts:build", - "wasm:build": "cd postgres-pglite && ./build-with-docker.sh; cd .. && mkdir -p ./packages/pglite/release/ && cp ./postgres-pglite/dist/bin/pglite.* ./packages/pglite/release/ && cp ./postgres-pglite/dist/extensions/*.tar.gz ./packages/pglite/release/", + "wasm:build": "cd postgres-pglite && ./build-with-docker.sh && cd .. && mkdir -p ./packages/pglite/release/ && cp ./postgres-pglite/dist/bin/pglite.* ./packages/pglite/release/ && cp ./postgres-pglite/dist/extensions/*.tar.gz ./packages/pglite/release/", "wasm:build:debug": "DEBUG=true pnpm wasm:build", "build:all": "pnpm wasm:build && pnpm ts:build", "build:all:debug": "DEBUG=true pnpm build:all" diff --git a/postgres-pglite b/postgres-pglite index 1e4f3c000..dccf2a30f 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 1e4f3c0002ace80f5d75ee5789d3249ea65e4064 +Subproject commit dccf2a30f079d70a042059d51713d75b1d65747d From b76e71c0864130be8f96236765788c1a03b5371d Mon Sep 17 00:00:00 2001 From: tudor Date: Fri, 3 Oct 2025 22:08:17 +0200 Subject: [PATCH 34/79] submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index dccf2a30f..4ed926dd0 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit dccf2a30f079d70a042059d51713d75b1d65747d +Subproject commit 4ed926dd0c869999afe40e93eb43b1de23fcf611 From 7a68dabdb4320c4b1d87a208a0bc545938b98633 Mon Sep 17 00:00:00 2001 From: tudor Date: Fri, 3 Oct 2025 22:26:38 +0200 Subject: [PATCH 35/79] postgis tests --- packages/pglite/tests/postgis.test.ts | 34 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/packages/pglite/tests/postgis.test.ts b/packages/pglite/tests/postgis.test.ts index f33ec14d5..fdf388626 100644 --- a/packages/pglite/tests/postgis.test.ts +++ b/packages/pglite/tests/postgis.test.ts @@ -33,36 +33,38 @@ await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { location GEOGRAPHY(POINT, 4326) ); `) - await pg.exec(`INSERT INTO vehicle_location VALUES + const inserted = await pg.query(`INSERT INTO vehicle_location VALUES ('2023-05-29 20:00:00', 1, 'POINT(15.3672 -87.7231)'), ('2023-05-30 20:00:00', 1, 'POINT(15.3652 -80.7331)'), ('2023-05-31 20:00:00', 1, 'POINT(15.2672 -85.7431)');`) - + expect(inserted.affectedRows).toEqual(3) }), - it('cities', async () => { - const pg = new PGlite({ - extensions: { - postgis, - }, - defaultDataTransferContainer, - }) + it('cities', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + defaultDataTransferContainer, + }) - await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') - await pg.exec(` + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + await pg.exec(` CREATE TABLE cities ( id SERIAL PRIMARY KEY, name VARCHAR(100), location GEOMETRY(Point, 4326) ); `) - await pg.exec(`INSERT INTO cities (name, location) + const inserted = await pg.query(`INSERT INTO cities (name, location) VALUES ('New York', ST_GeomFromText('POINT(-74.0060 40.7128)', 4326)), ('Los Angeles', ST_GeomFromText('POINT(-118.2437 34.0522)', 4326)), ('Chicago', ST_GeomFromText('POINT(-87.6298 41.8781)', 4326));`) - await pg.exec(`WITH state_boundary AS ( + expect(inserted.affectedRows).toEqual(3) + + const cities = await pg.query(`WITH state_boundary AS ( SELECT ST_GeomFromText( 'POLYGON((-91 36, -91 43, -87 43, -87 36, -91 36))', 4326 ) AS geom @@ -71,6 +73,10 @@ SELECT c.name FROM cities c, state_boundary s WHERE ST_Within(c.location, s.geom);`) - }) + expect(cities.affectedRows).toBe(0) + expect(cities.rows[0]).toEqual({ + name: 'Chicago', + }) + }) }) }) From e260de31d5ddc657d48b6f3f675cf2939468afd0 Mon Sep 17 00:00:00 2001 From: tudor Date: Fri, 3 Oct 2025 22:29:51 +0200 Subject: [PATCH 36/79] submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 4ed926dd0..439813d23 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 4ed926dd0c869999afe40e93eb43b1de23fcf611 +Subproject commit 439813d2314978d07f1aa341fbd4373834dd1bf0 From dbae0ffdcb66a8990157854f95641aa59b1571fc Mon Sep 17 00:00:00 2001 From: tudor Date: Fri, 3 Oct 2025 22:47:13 +0200 Subject: [PATCH 37/79] submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 439813d23..99e377d5f 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 439813d2314978d07f1aa341fbd4373834dd1bf0 +Subproject commit 99e377d5fe5c031fb566dae098b7ab4d963a9cbe From b2b9d40058723f39d3c2f430b2ccfa9c21442100 Mon Sep 17 00:00:00 2001 From: tudor Date: Sat, 4 Oct 2025 18:53:20 +0200 Subject: [PATCH 38/79] submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 99e377d5f..099afd57c 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 99e377d5fe5c031fb566dae098b7ab4d963a9cbe +Subproject commit 099afd57c7f41b623051647ad0cf7b8b32d1983f From dff572784364416129f4e89ee09acf975fa7fb99 Mon Sep 17 00:00:00 2001 From: tudor Date: Sat, 4 Oct 2025 19:01:15 +0200 Subject: [PATCH 39/79] submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 099afd57c..3fff75a03 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 099afd57c7f41b623051647ad0cf7b8b32d1983f +Subproject commit 3fff75a03e48f08c44af51faf02eb46a78927142 From b7709a334c32942100f96067d92d0842bc96f916 Mon Sep 17 00:00:00 2001 From: tudor Date: Sun, 5 Oct 2025 09:46:45 +0200 Subject: [PATCH 40/79] submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 3fff75a03..5a86cb0ab 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 3fff75a03e48f08c44af51faf02eb46a78927142 +Subproject commit 5a86cb0ab29ad9dbf1350b3b51632e76685a9c1a From aea919f8d02ef89c339820f134d0cedcd0db33e0 Mon Sep 17 00:00:00 2001 From: tudor Date: Sun, 5 Oct 2025 20:20:13 +0200 Subject: [PATCH 41/79] a more complex postgis test; update submodule; --- packages/pglite/tests/postgis.test.ts | 80 +++++++++++++++++++++++++++ postgres-pglite | 2 +- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/packages/pglite/tests/postgis.test.ts b/packages/pglite/tests/postgis.test.ts index fdf388626..ada4e979d 100644 --- a/packages/pglite/tests/postgis.test.ts +++ b/packages/pglite/tests/postgis.test.ts @@ -79,4 +79,84 @@ WHERE ST_Within(c.location, s.geom);`) }) }) }) + it('complex1', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + defaultDataTransferContainer, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + + const x = await pg.exec(` + -- Create test schema +-- CREATE SCHEMA IF NOT EXISTS postgis_test; +-- SET search_path TO postgis_test; + +-- Create a table with geometry columns +CREATE TABLE cities ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + population INTEGER, + geom GEOMETRY(Point, 4326) +);` + ) + + const y = await pg.exec(` +CREATE TABLE rivers ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + geom GEOMETRY(LineString, 4326) +); + +-- Insert sample data +INSERT INTO cities (name, population, geom) VALUES +('Paris', 2148000, ST_SetSRID(ST_MakePoint(2.3522, 48.8566), 4326)), +('Berlin', 3769000, ST_SetSRID(ST_MakePoint(13.4050, 52.5200), 4326)), +('London', 8982000, ST_SetSRID(ST_MakePoint(-0.1276, 51.5072), 4326)), +('Amsterdam', 872757, ST_SetSRID(ST_MakePoint(4.9041, 52.3676), 4326)); + +INSERT INTO rivers (name, geom) VALUES +('Seine', ST_SetSRID(ST_MakeLine(ARRAY[ + ST_MakePoint(2.1, 48.8), + ST_MakePoint(2.35, 48.85), + ST_MakePoint(2.45, 48.9) +]), 4326)), +('Spree', ST_SetSRID(ST_MakeLine(ARRAY[ + ST_MakePoint(13.1, 52.4), + ST_MakePoint(13.35, 52.5), + ST_MakePoint(13.45, 52.52) +]), 4326)); + +-- Create spatial index +CREATE INDEX idx_cities_geom ON cities USING GIST (geom); +CREATE INDEX idx_rivers_geom ON rivers USING GIST (geom); + +-- Query: Find cities within 10 km of any river +SELECT + c.name AS city, + r.name AS river, + ST_Distance(c.geom::geography, r.geom::geography) AS distance_km +FROM cities c +JOIN rivers r +ON ST_DWithin(c.geom::geography, r.geom::geography, 10000) +ORDER BY distance_km; + +-- Query: Compute buffered area around each river and intersecting cities +SELECT + r.name AS river_name, + COUNT(c.id) AS num_cities_intersecting, + ST_Area(ST_Transform(ST_Buffer(r.geom::geography, 5000), 3857)) / 1e6 AS buffer_area_sqkm +FROM rivers r +LEFT JOIN cities c +ON ST_Intersects(ST_Buffer(r.geom::geography, 5000), c.geom) +GROUP BY r.name; + +-- Cleanup test schema +-- DROP SCHEMA postgis_test CASCADE; +`) + + console.log(x) + + }) }) diff --git a/postgres-pglite b/postgres-pglite index f6c016d80..de4b1271d 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit f6c016d8027559b137b5d4205cdfe4c6669edad2 +Subproject commit de4b1271d94d5a625d60da4996ba3e7dd9ef78a8 From 9bb972b51867bddeecc0225bb5a2c6ef1a381797 Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 12:05:03 +0200 Subject: [PATCH 42/79] cleanup; update submodule --- packages/pglite/src/postgresMod.ts | 2 -- packages/pglite/vitest.config.ts | 4 ++-- postgres-pglite | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/pglite/src/postgresMod.ts b/packages/pglite/src/postgresMod.ts index 8bba4130f..cfe5c0de4 100644 --- a/packages/pglite/src/postgresMod.ts +++ b/packages/pglite/src/postgresMod.ts @@ -28,10 +28,8 @@ export interface PostgresMod _pgl_initdb: () => number _pgl_backend: () => void _pgl_shutdown: () => void - _get_channel: () => number _interactive_write: (msgLength: number) => void _interactive_one: (length: number, peek: number) => void - _interactive_read: () => number _set_read_write_cbs: (read_cb: number, write_cb: number) => void } diff --git a/packages/pglite/vitest.config.ts b/packages/pglite/vitest.config.ts index ecbe2988a..b51b2c7ad 100644 --- a/packages/pglite/vitest.config.ts +++ b/packages/pglite/vitest.config.ts @@ -6,8 +6,8 @@ export default defineConfig({ dir: './tests', watch: false, typecheck: { enabled: true }, - testTimeout: 30000, - hookTimeout: 30000, + // testTimeout: 30000, + // hookTimeout: 30000, include: ['**/*.{test,test.web}.{js,ts}'], server: { deps: { diff --git a/postgres-pglite b/postgres-pglite index fd02949cd..a4013b504 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit fd02949cd5ac26ecfb0a15ef8f78bb54ca803cb4 +Subproject commit a4013b504e9cfad59fc2d6db3d41267aba326c3c From bda4b68f5354703a09bb82bf979680540c58f2a4 Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 14:07:52 +0200 Subject: [PATCH 43/79] cleanupof additional syncs; update submodule --- packages/pglite/src/base.ts | 60 +++++++++++++++++-------------------- postgres-pglite | 2 +- 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/packages/pglite/src/base.ts b/packages/pglite/src/base.ts index f06bb100a..4beb16392 100644 --- a/packages/pglite/src/base.ts +++ b/packages/pglite/src/base.ts @@ -242,9 +242,7 @@ export abstract class BasePGlite serializeProtocol.describe({ type: 'S' }), options, ) - ).messages, - ...(await this.#execProtocolNoSync(serializeProtocol.sync(), options)) - .messages, + ).messages ]) const values = params.map((param, i) => { @@ -368,46 +366,19 @@ export abstract class BasePGlite query: string, options?: QueryOptions, ): Promise { + let messages = [] try { await this.#execProtocolNoSync( serializeProtocol.parse({ text: query, types: options?.paramTypes }), options, ) - const messages = ( + messages = ( await this.#execProtocolNoSync( serializeProtocol.describe({ type: 'S' }), options, ) ).messages - - messages.push( - ...(await this.#execProtocolNoSync(serializeProtocol.sync(), options)) - .messages, - ) - - const paramDescription = messages.find( - (msg): msg is ParameterDescriptionMessage => - msg.name === 'parameterDescription', - ) - const resultDescription = messages.find( - (msg): msg is RowDescriptionMessage => msg.name === 'rowDescription', - ) - - const queryParams = - paramDescription?.dataTypeIDs.map((dataTypeID) => ({ - dataTypeID, - serializer: this.serializers[dataTypeID], - })) ?? [] - - const resultFields = - resultDescription?.fields.map((field) => ({ - name: field.name, - dataTypeID: field.dataTypeID, - parser: this.parsers[field.dataTypeID], - })) ?? [] - - return { queryParams, resultFields } } catch (e) { if (e instanceof DatabaseError) { const pgError = makePGliteError({ @@ -420,8 +391,31 @@ export abstract class BasePGlite } throw e } finally { - await this.#execProtocolNoSync(serializeProtocol.sync(), options) + messages.push(...(await this.#execProtocolNoSync(serializeProtocol.sync(), options)).messages) } + + const paramDescription = messages.find( + (msg): msg is ParameterDescriptionMessage => + msg.name === 'parameterDescription', + ) + const resultDescription = messages.find( + (msg): msg is RowDescriptionMessage => msg.name === 'rowDescription', + ) + + const queryParams = + paramDescription?.dataTypeIDs.map((dataTypeID) => ({ + dataTypeID, + serializer: this.serializers[dataTypeID], + })) ?? [] + + const resultFields = + resultDescription?.fields.map((field) => ({ + name: field.name, + dataTypeID: field.dataTypeID, + parser: this.parsers[field.dataTypeID], + })) ?? [] + + return { queryParams, resultFields } } /** diff --git a/postgres-pglite b/postgres-pglite index a4013b504..5452dfc6f 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit a4013b504e9cfad59fc2d6db3d41267aba326c3c +Subproject commit 5452dfc6f86092d775c6599c40e3c491ab547d98 From 43ea96c77595312de47c466c6c7d208617575968 Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 14:11:11 +0200 Subject: [PATCH 44/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 5452dfc6f..219bdc609 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 5452dfc6f86092d775c6599c40e3c491ab547d98 +Subproject commit 219bdc609641caf4bce78c350fd76ca0de976eea From f95eb519580dbd22c5d291b008266a909f1c52d7 Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 14:14:27 +0200 Subject: [PATCH 45/79] Revert "update submodule" This reverts commit 43ea96c77595312de47c466c6c7d208617575968. --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 219bdc609..5452dfc6f 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 219bdc609641caf4bce78c350fd76ca0de976eea +Subproject commit 5452dfc6f86092d775c6599c40e3c491ab547d98 From 6bb206159558b5539b2c41b30b37f03cac22e814 Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 14:14:43 +0200 Subject: [PATCH 46/79] revert submodule update --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 5452dfc6f..219bdc609 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 5452dfc6f86092d775c6599c40e3c491ab547d98 +Subproject commit 219bdc609641caf4bce78c350fd76ca0de976eea From dba962b03b87da055bce797dfb3cf33ba0d5aadc Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 14:20:38 +0200 Subject: [PATCH 47/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 219bdc609..5452dfc6f 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 219bdc609641caf4bce78c350fd76ca0de976eea +Subproject commit 5452dfc6f86092d775c6599c40e3c491ab547d98 From c5af87d4e7b84dbd831399ad130aa0aae4f9ca25 Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 14:21:23 +0200 Subject: [PATCH 48/79] stylecheck --- packages/pglite/src/base.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/pglite/src/base.ts b/packages/pglite/src/base.ts index 4beb16392..09771d9a4 100644 --- a/packages/pglite/src/base.ts +++ b/packages/pglite/src/base.ts @@ -242,7 +242,7 @@ export abstract class BasePGlite serializeProtocol.describe({ type: 'S' }), options, ) - ).messages + ).messages, ]) const values = params.map((param, i) => { @@ -391,7 +391,10 @@ export abstract class BasePGlite } throw e } finally { - messages.push(...(await this.#execProtocolNoSync(serializeProtocol.sync(), options)).messages) + messages.push( + ...(await this.#execProtocolNoSync(serializeProtocol.sync(), options)) + .messages, + ) } const paramDescription = messages.find( From 1b953e4ed7870dd53b502981d5dd0dce119520aa Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 14:22:55 +0200 Subject: [PATCH 49/79] undo --- packages/pglite/vitest.config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pglite/vitest.config.ts b/packages/pglite/vitest.config.ts index b51b2c7ad..ecbe2988a 100644 --- a/packages/pglite/vitest.config.ts +++ b/packages/pglite/vitest.config.ts @@ -6,8 +6,8 @@ export default defineConfig({ dir: './tests', watch: false, typecheck: { enabled: true }, - // testTimeout: 30000, - // hookTimeout: 30000, + testTimeout: 30000, + hookTimeout: 30000, include: ['**/*.{test,test.web}.{js,ts}'], server: { deps: { From cddcf79854aefebc2a8968fa8200a78db1b2f431 Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 14:38:29 +0200 Subject: [PATCH 50/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 5452dfc6f..219bdc609 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 5452dfc6f86092d775c6599c40e3c491ab547d98 +Subproject commit 219bdc609641caf4bce78c350fd76ca0de976eea From 31f7b7c5fcee638e4852a63f862a319f5b727f13 Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 14:40:42 +0200 Subject: [PATCH 51/79] stylecheck --- packages/pglite/tests/postgis.test.ts | 30 ++++++++++++--------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/packages/pglite/tests/postgis.test.ts b/packages/pglite/tests/postgis.test.ts index ada4e979d..63ec77a01 100644 --- a/packages/pglite/tests/postgis.test.ts +++ b/packages/pglite/tests/postgis.test.ts @@ -79,16 +79,16 @@ WHERE ST_Within(c.location, s.geom);`) }) }) }) - it('complex1', async () => { - const pg = new PGlite({ - extensions: { - postgis, - }, - defaultDataTransferContainer, - }) - await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') - - const x = await pg.exec(` + it('complex1', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + defaultDataTransferContainer, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + + await pg.exec(` -- Create test schema -- CREATE SCHEMA IF NOT EXISTS postgis_test; -- SET search_path TO postgis_test; @@ -99,10 +99,9 @@ CREATE TABLE cities ( name TEXT NOT NULL, population INTEGER, geom GEOMETRY(Point, 4326) -);` - ) +);`) - const y = await pg.exec(` + await pg.exec(` CREATE TABLE rivers ( id SERIAL PRIMARY KEY, name TEXT NOT NULL, @@ -155,8 +154,5 @@ GROUP BY r.name; -- Cleanup test schema -- DROP SCHEMA postgis_test CASCADE; `) - - console.log(x) - - }) + }) }) From 28b2ae80fd92dde4a6455ef97f2706fdfc933730 Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 15:59:24 +0200 Subject: [PATCH 52/79] comment out complex1 postgis test --- packages/pglite/tests/postgis.test.ts | 152 +++++++++++++------------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/packages/pglite/tests/postgis.test.ts b/packages/pglite/tests/postgis.test.ts index 63ec77a01..29f679f58 100644 --- a/packages/pglite/tests/postgis.test.ts +++ b/packages/pglite/tests/postgis.test.ts @@ -79,80 +79,80 @@ WHERE ST_Within(c.location, s.geom);`) }) }) }) - it('complex1', async () => { - const pg = new PGlite({ - extensions: { - postgis, - }, - defaultDataTransferContainer, - }) - await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') - - await pg.exec(` - -- Create test schema --- CREATE SCHEMA IF NOT EXISTS postgis_test; --- SET search_path TO postgis_test; - --- Create a table with geometry columns -CREATE TABLE cities ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - population INTEGER, - geom GEOMETRY(Point, 4326) -);`) - - await pg.exec(` -CREATE TABLE rivers ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - geom GEOMETRY(LineString, 4326) -); - --- Insert sample data -INSERT INTO cities (name, population, geom) VALUES -('Paris', 2148000, ST_SetSRID(ST_MakePoint(2.3522, 48.8566), 4326)), -('Berlin', 3769000, ST_SetSRID(ST_MakePoint(13.4050, 52.5200), 4326)), -('London', 8982000, ST_SetSRID(ST_MakePoint(-0.1276, 51.5072), 4326)), -('Amsterdam', 872757, ST_SetSRID(ST_MakePoint(4.9041, 52.3676), 4326)); - -INSERT INTO rivers (name, geom) VALUES -('Seine', ST_SetSRID(ST_MakeLine(ARRAY[ - ST_MakePoint(2.1, 48.8), - ST_MakePoint(2.35, 48.85), - ST_MakePoint(2.45, 48.9) -]), 4326)), -('Spree', ST_SetSRID(ST_MakeLine(ARRAY[ - ST_MakePoint(13.1, 52.4), - ST_MakePoint(13.35, 52.5), - ST_MakePoint(13.45, 52.52) -]), 4326)); - --- Create spatial index -CREATE INDEX idx_cities_geom ON cities USING GIST (geom); -CREATE INDEX idx_rivers_geom ON rivers USING GIST (geom); - --- Query: Find cities within 10 km of any river -SELECT - c.name AS city, - r.name AS river, - ST_Distance(c.geom::geography, r.geom::geography) AS distance_km -FROM cities c -JOIN rivers r -ON ST_DWithin(c.geom::geography, r.geom::geography, 10000) -ORDER BY distance_km; - --- Query: Compute buffered area around each river and intersecting cities -SELECT - r.name AS river_name, - COUNT(c.id) AS num_cities_intersecting, - ST_Area(ST_Transform(ST_Buffer(r.geom::geography, 5000), 3857)) / 1e6 AS buffer_area_sqkm -FROM rivers r -LEFT JOIN cities c -ON ST_Intersects(ST_Buffer(r.geom::geography, 5000), c.geom) -GROUP BY r.name; - --- Cleanup test schema --- DROP SCHEMA postgis_test CASCADE; -`) - }) +// it('complex1', async () => { +// const pg = new PGlite({ +// extensions: { +// postgis, +// }, +// defaultDataTransferContainer, +// }) +// await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + +// await pg.exec(` +// -- Create test schema +// -- CREATE SCHEMA IF NOT EXISTS postgis_test; +// -- SET search_path TO postgis_test; + +// -- Create a table with geometry columns +// CREATE TABLE cities ( +// id SERIAL PRIMARY KEY, +// name TEXT NOT NULL, +// population INTEGER, +// geom GEOMETRY(Point, 4326) +// );`) + +// await pg.exec(` +// CREATE TABLE rivers ( +// id SERIAL PRIMARY KEY, +// name TEXT NOT NULL, +// geom GEOMETRY(LineString, 4326) +// ); + +// -- Insert sample data +// INSERT INTO cities (name, population, geom) VALUES +// ('Paris', 2148000, ST_SetSRID(ST_MakePoint(2.3522, 48.8566), 4326)), +// ('Berlin', 3769000, ST_SetSRID(ST_MakePoint(13.4050, 52.5200), 4326)), +// ('London', 8982000, ST_SetSRID(ST_MakePoint(-0.1276, 51.5072), 4326)), +// ('Amsterdam', 872757, ST_SetSRID(ST_MakePoint(4.9041, 52.3676), 4326)); + +// INSERT INTO rivers (name, geom) VALUES +// ('Seine', ST_SetSRID(ST_MakeLine(ARRAY[ +// ST_MakePoint(2.1, 48.8), +// ST_MakePoint(2.35, 48.85), +// ST_MakePoint(2.45, 48.9) +// ]), 4326)), +// ('Spree', ST_SetSRID(ST_MakeLine(ARRAY[ +// ST_MakePoint(13.1, 52.4), +// ST_MakePoint(13.35, 52.5), +// ST_MakePoint(13.45, 52.52) +// ]), 4326)); + +// -- Create spatial index +// CREATE INDEX idx_cities_geom ON cities USING GIST (geom); +// CREATE INDEX idx_rivers_geom ON rivers USING GIST (geom); + +// -- Query: Find cities within 10 km of any river +// SELECT +// c.name AS city, +// r.name AS river, +// ST_Distance(c.geom::geography, r.geom::geography) AS distance_km +// FROM cities c +// JOIN rivers r +// ON ST_DWithin(c.geom::geography, r.geom::geography, 10000) +// ORDER BY distance_km; + +// -- Query: Compute buffered area around each river and intersecting cities +// SELECT +// r.name AS river_name, +// COUNT(c.id) AS num_cities_intersecting, +// ST_Area(ST_Transform(ST_Buffer(r.geom::geography, 5000), 3857)) / 1e6 AS buffer_area_sqkm +// FROM rivers r +// LEFT JOIN cities c +// ON ST_Intersects(ST_Buffer(r.geom::geography, 5000), c.geom) +// GROUP BY r.name; + +// -- Cleanup test schema +// -- DROP SCHEMA postgis_test CASCADE; +// `) +// }) }) From f850e9ebcb13d8ceba2447ef128c09ebd6c9a0a5 Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 16:53:43 +0200 Subject: [PATCH 53/79] style --- packages/pglite/tests/postgis.test.ts | 152 +++++++++++++------------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/packages/pglite/tests/postgis.test.ts b/packages/pglite/tests/postgis.test.ts index 29f679f58..0af92c38f 100644 --- a/packages/pglite/tests/postgis.test.ts +++ b/packages/pglite/tests/postgis.test.ts @@ -79,80 +79,80 @@ WHERE ST_Within(c.location, s.geom);`) }) }) }) -// it('complex1', async () => { -// const pg = new PGlite({ -// extensions: { -// postgis, -// }, -// defaultDataTransferContainer, -// }) -// await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') - -// await pg.exec(` -// -- Create test schema -// -- CREATE SCHEMA IF NOT EXISTS postgis_test; -// -- SET search_path TO postgis_test; - -// -- Create a table with geometry columns -// CREATE TABLE cities ( -// id SERIAL PRIMARY KEY, -// name TEXT NOT NULL, -// population INTEGER, -// geom GEOMETRY(Point, 4326) -// );`) - -// await pg.exec(` -// CREATE TABLE rivers ( -// id SERIAL PRIMARY KEY, -// name TEXT NOT NULL, -// geom GEOMETRY(LineString, 4326) -// ); - -// -- Insert sample data -// INSERT INTO cities (name, population, geom) VALUES -// ('Paris', 2148000, ST_SetSRID(ST_MakePoint(2.3522, 48.8566), 4326)), -// ('Berlin', 3769000, ST_SetSRID(ST_MakePoint(13.4050, 52.5200), 4326)), -// ('London', 8982000, ST_SetSRID(ST_MakePoint(-0.1276, 51.5072), 4326)), -// ('Amsterdam', 872757, ST_SetSRID(ST_MakePoint(4.9041, 52.3676), 4326)); - -// INSERT INTO rivers (name, geom) VALUES -// ('Seine', ST_SetSRID(ST_MakeLine(ARRAY[ -// ST_MakePoint(2.1, 48.8), -// ST_MakePoint(2.35, 48.85), -// ST_MakePoint(2.45, 48.9) -// ]), 4326)), -// ('Spree', ST_SetSRID(ST_MakeLine(ARRAY[ -// ST_MakePoint(13.1, 52.4), -// ST_MakePoint(13.35, 52.5), -// ST_MakePoint(13.45, 52.52) -// ]), 4326)); - -// -- Create spatial index -// CREATE INDEX idx_cities_geom ON cities USING GIST (geom); -// CREATE INDEX idx_rivers_geom ON rivers USING GIST (geom); - -// -- Query: Find cities within 10 km of any river -// SELECT -// c.name AS city, -// r.name AS river, -// ST_Distance(c.geom::geography, r.geom::geography) AS distance_km -// FROM cities c -// JOIN rivers r -// ON ST_DWithin(c.geom::geography, r.geom::geography, 10000) -// ORDER BY distance_km; - -// -- Query: Compute buffered area around each river and intersecting cities -// SELECT -// r.name AS river_name, -// COUNT(c.id) AS num_cities_intersecting, -// ST_Area(ST_Transform(ST_Buffer(r.geom::geography, 5000), 3857)) / 1e6 AS buffer_area_sqkm -// FROM rivers r -// LEFT JOIN cities c -// ON ST_Intersects(ST_Buffer(r.geom::geography, 5000), c.geom) -// GROUP BY r.name; - -// -- Cleanup test schema -// -- DROP SCHEMA postgis_test CASCADE; -// `) -// }) + // it('complex1', async () => { + // const pg = new PGlite({ + // extensions: { + // postgis, + // }, + // defaultDataTransferContainer, + // }) + // await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + + // await pg.exec(` + // -- Create test schema + // -- CREATE SCHEMA IF NOT EXISTS postgis_test; + // -- SET search_path TO postgis_test; + + // -- Create a table with geometry columns + // CREATE TABLE cities ( + // id SERIAL PRIMARY KEY, + // name TEXT NOT NULL, + // population INTEGER, + // geom GEOMETRY(Point, 4326) + // );`) + + // await pg.exec(` + // CREATE TABLE rivers ( + // id SERIAL PRIMARY KEY, + // name TEXT NOT NULL, + // geom GEOMETRY(LineString, 4326) + // ); + + // -- Insert sample data + // INSERT INTO cities (name, population, geom) VALUES + // ('Paris', 2148000, ST_SetSRID(ST_MakePoint(2.3522, 48.8566), 4326)), + // ('Berlin', 3769000, ST_SetSRID(ST_MakePoint(13.4050, 52.5200), 4326)), + // ('London', 8982000, ST_SetSRID(ST_MakePoint(-0.1276, 51.5072), 4326)), + // ('Amsterdam', 872757, ST_SetSRID(ST_MakePoint(4.9041, 52.3676), 4326)); + + // INSERT INTO rivers (name, geom) VALUES + // ('Seine', ST_SetSRID(ST_MakeLine(ARRAY[ + // ST_MakePoint(2.1, 48.8), + // ST_MakePoint(2.35, 48.85), + // ST_MakePoint(2.45, 48.9) + // ]), 4326)), + // ('Spree', ST_SetSRID(ST_MakeLine(ARRAY[ + // ST_MakePoint(13.1, 52.4), + // ST_MakePoint(13.35, 52.5), + // ST_MakePoint(13.45, 52.52) + // ]), 4326)); + + // -- Create spatial index + // CREATE INDEX idx_cities_geom ON cities USING GIST (geom); + // CREATE INDEX idx_rivers_geom ON rivers USING GIST (geom); + + // -- Query: Find cities within 10 km of any river + // SELECT + // c.name AS city, + // r.name AS river, + // ST_Distance(c.geom::geography, r.geom::geography) AS distance_km + // FROM cities c + // JOIN rivers r + // ON ST_DWithin(c.geom::geography, r.geom::geography, 10000) + // ORDER BY distance_km; + + // -- Query: Compute buffered area around each river and intersecting cities + // SELECT + // r.name AS river_name, + // COUNT(c.id) AS num_cities_intersecting, + // ST_Area(ST_Transform(ST_Buffer(r.geom::geography, 5000), 3857)) / 1e6 AS buffer_area_sqkm + // FROM rivers r + // LEFT JOIN cities c + // ON ST_Intersects(ST_Buffer(r.geom::geography, 5000), c.geom) + // GROUP BY r.name; + + // -- Cleanup test schema + // -- DROP SCHEMA postgis_test CASCADE; + // `) + // }) }) From 7546a11d1f2439c91763c013eb59dc1aff420dbb Mon Sep 17 00:00:00 2001 From: tudor Date: Fri, 7 Nov 2025 16:59:26 +0100 Subject: [PATCH 54/79] reenable failing tests --- packages/pglite/tests/postgis.test.ts | 152 +++++++++++++------------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/packages/pglite/tests/postgis.test.ts b/packages/pglite/tests/postgis.test.ts index 0af92c38f..1698ce17d 100644 --- a/packages/pglite/tests/postgis.test.ts +++ b/packages/pglite/tests/postgis.test.ts @@ -79,80 +79,80 @@ WHERE ST_Within(c.location, s.geom);`) }) }) }) - // it('complex1', async () => { - // const pg = new PGlite({ - // extensions: { - // postgis, - // }, - // defaultDataTransferContainer, - // }) - // await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') - - // await pg.exec(` - // -- Create test schema - // -- CREATE SCHEMA IF NOT EXISTS postgis_test; - // -- SET search_path TO postgis_test; - - // -- Create a table with geometry columns - // CREATE TABLE cities ( - // id SERIAL PRIMARY KEY, - // name TEXT NOT NULL, - // population INTEGER, - // geom GEOMETRY(Point, 4326) - // );`) - - // await pg.exec(` - // CREATE TABLE rivers ( - // id SERIAL PRIMARY KEY, - // name TEXT NOT NULL, - // geom GEOMETRY(LineString, 4326) - // ); - - // -- Insert sample data - // INSERT INTO cities (name, population, geom) VALUES - // ('Paris', 2148000, ST_SetSRID(ST_MakePoint(2.3522, 48.8566), 4326)), - // ('Berlin', 3769000, ST_SetSRID(ST_MakePoint(13.4050, 52.5200), 4326)), - // ('London', 8982000, ST_SetSRID(ST_MakePoint(-0.1276, 51.5072), 4326)), - // ('Amsterdam', 872757, ST_SetSRID(ST_MakePoint(4.9041, 52.3676), 4326)); - - // INSERT INTO rivers (name, geom) VALUES - // ('Seine', ST_SetSRID(ST_MakeLine(ARRAY[ - // ST_MakePoint(2.1, 48.8), - // ST_MakePoint(2.35, 48.85), - // ST_MakePoint(2.45, 48.9) - // ]), 4326)), - // ('Spree', ST_SetSRID(ST_MakeLine(ARRAY[ - // ST_MakePoint(13.1, 52.4), - // ST_MakePoint(13.35, 52.5), - // ST_MakePoint(13.45, 52.52) - // ]), 4326)); - - // -- Create spatial index - // CREATE INDEX idx_cities_geom ON cities USING GIST (geom); - // CREATE INDEX idx_rivers_geom ON rivers USING GIST (geom); - - // -- Query: Find cities within 10 km of any river - // SELECT - // c.name AS city, - // r.name AS river, - // ST_Distance(c.geom::geography, r.geom::geography) AS distance_km - // FROM cities c - // JOIN rivers r - // ON ST_DWithin(c.geom::geography, r.geom::geography, 10000) - // ORDER BY distance_km; - - // -- Query: Compute buffered area around each river and intersecting cities - // SELECT - // r.name AS river_name, - // COUNT(c.id) AS num_cities_intersecting, - // ST_Area(ST_Transform(ST_Buffer(r.geom::geography, 5000), 3857)) / 1e6 AS buffer_area_sqkm - // FROM rivers r - // LEFT JOIN cities c - // ON ST_Intersects(ST_Buffer(r.geom::geography, 5000), c.geom) - // GROUP BY r.name; - - // -- Cleanup test schema - // -- DROP SCHEMA postgis_test CASCADE; - // `) - // }) + it('complex1', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + defaultDataTransferContainer, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + + await pg.exec(` + -- Create test schema + -- CREATE SCHEMA IF NOT EXISTS postgis_test; + -- SET search_path TO postgis_test; + + -- Create a table with geometry columns + CREATE TABLE cities ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + population INTEGER, + geom GEOMETRY(Point, 4326) + );`) + + await pg.exec(` + CREATE TABLE rivers ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + geom GEOMETRY(LineString, 4326) + ); + + -- Insert sample data + INSERT INTO cities (name, population, geom) VALUES + ('Paris', 2148000, ST_SetSRID(ST_MakePoint(2.3522, 48.8566), 4326)), + ('Berlin', 3769000, ST_SetSRID(ST_MakePoint(13.4050, 52.5200), 4326)), + ('London', 8982000, ST_SetSRID(ST_MakePoint(-0.1276, 51.5072), 4326)), + ('Amsterdam', 872757, ST_SetSRID(ST_MakePoint(4.9041, 52.3676), 4326)); + + INSERT INTO rivers (name, geom) VALUES + ('Seine', ST_SetSRID(ST_MakeLine(ARRAY[ + ST_MakePoint(2.1, 48.8), + ST_MakePoint(2.35, 48.85), + ST_MakePoint(2.45, 48.9) + ]), 4326)), + ('Spree', ST_SetSRID(ST_MakeLine(ARRAY[ + ST_MakePoint(13.1, 52.4), + ST_MakePoint(13.35, 52.5), + ST_MakePoint(13.45, 52.52) + ]), 4326)); + + -- Create spatial index + CREATE INDEX idx_cities_geom ON cities USING GIST (geom); + CREATE INDEX idx_rivers_geom ON rivers USING GIST (geom); + + -- Query: Find cities within 10 km of any river + SELECT + c.name AS city, + r.name AS river, + ST_Distance(c.geom::geography, r.geom::geography) AS distance_km + FROM cities c + JOIN rivers r + ON ST_DWithin(c.geom::geography, r.geom::geography, 10000) + ORDER BY distance_km; + + -- Query: Compute buffered area around each river and intersecting cities + SELECT + r.name AS river_name, + COUNT(c.id) AS num_cities_intersecting, + ST_Area(ST_Transform(ST_Buffer(r.geom::geography, 5000), 3857)) / 1e6 AS buffer_area_sqkm + FROM rivers r + LEFT JOIN cities c + ON ST_Intersects(ST_Buffer(r.geom::geography, 5000), c.geom) + GROUP BY r.name; + + -- Cleanup test schema + -- DROP SCHEMA postgis_test CASCADE; + `) + }) }) From cbf165496b8017005ab83499a9100755c05ba861 Mon Sep 17 00:00:00 2001 From: tudor Date: Fri, 7 Nov 2025 16:59:34 +0100 Subject: [PATCH 55/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index a1a381bd8..eadfeabf7 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit a1a381bd800ae960653ebf5919606d826cd3fdfe +Subproject commit eadfeabf753b1ed81adfdbeaa72477ed0743fab4 From deafe66cabcdfc625c9ab44bf98ba2995e415514 Mon Sep 17 00:00:00 2001 From: tudor Date: Fri, 7 Nov 2025 17:50:09 +0100 Subject: [PATCH 56/79] style --- packages/pglite/tests/postgis.test.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/pglite/tests/postgis.test.ts b/packages/pglite/tests/postgis.test.ts index 1698ce17d..58da4f4f8 100644 --- a/packages/pglite/tests/postgis.test.ts +++ b/packages/pglite/tests/postgis.test.ts @@ -79,16 +79,16 @@ WHERE ST_Within(c.location, s.geom);`) }) }) }) - it('complex1', async () => { - const pg = new PGlite({ - extensions: { - postgis, - }, - defaultDataTransferContainer, - }) - await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + it('complex1', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + defaultDataTransferContainer, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') - await pg.exec(` + await pg.exec(` -- Create test schema -- CREATE SCHEMA IF NOT EXISTS postgis_test; -- SET search_path TO postgis_test; @@ -101,7 +101,7 @@ WHERE ST_Within(c.location, s.geom);`) geom GEOMETRY(Point, 4326) );`) - await pg.exec(` + await pg.exec(` CREATE TABLE rivers ( id SERIAL PRIMARY KEY, name TEXT NOT NULL, @@ -154,5 +154,5 @@ WHERE ST_Within(c.location, s.geom);`) -- Cleanup test schema -- DROP SCHEMA postgis_test CASCADE; `) - }) + }) }) From 93f8d38a42504fe796d14b78f22ccc4eb07316fe Mon Sep 17 00:00:00 2001 From: tudor Date: Fri, 7 Nov 2025 21:26:04 +0100 Subject: [PATCH 57/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index eadfeabf7..270db96a1 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit eadfeabf753b1ed81adfdbeaa72477ed0743fab4 +Subproject commit 270db96a17e069951fb0bc64fbb409d9fefc3529 From 451639b3484ff5d00027ed4dff95bd24b94887ea Mon Sep 17 00:00:00 2001 From: tudor Date: Sat, 8 Nov 2025 09:24:05 +0100 Subject: [PATCH 58/79] simple area tests --- packages/pglite/tests/postgis.test.ts | 127 +++++++++++++++++++++++--- 1 file changed, 115 insertions(+), 12 deletions(-) diff --git a/packages/pglite/tests/postgis.test.ts b/packages/pglite/tests/postgis.test.ts index 58da4f4f8..cd4a36236 100644 --- a/packages/pglite/tests/postgis.test.ts +++ b/packages/pglite/tests/postgis.test.ts @@ -79,6 +79,121 @@ WHERE ST_Within(c.location, s.geom);`) }) }) }) + + it('areas', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + defaultDataTransferContainer, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + + const area1 = await pg.exec(` + select ST_Area(geom) sqft, + ST_Area(geom) * 0.3048 ^ 2 sqm + from ( + select 'SRID=2249;POLYGON((743238 2967416,743238 2967450, + 743265 2967450,743265.625 2967416,743238 2967416))' :: geometry geom + ) subquery;`) + + expect(area1).toEqual([ + { + rows: [ + { + sqft: 928.625, + sqm: 86.27208552, + }, + ], + fields: [ + { + name: 'sqft', + dataTypeID: 701, + }, + { + name: 'sqm', + dataTypeID: 701, + }, + ], + affectedRows: 0, + }, + ]) + + const area2 = await pg.exec(` + select ST_Area(geom) sqft, + ST_Area(ST_Transform(geom, 26986)) As sqm + from ( + select + 'SRID=2249;POLYGON((743238 2967416,743238 2967450, + 743265 2967450,743265.625 2967416,743238 2967416))' :: geometry geom + ) subquery; + + -- Cleanup test schema + -- DROP SCHEMA postgis_test CASCADE; + `) + + expect(area2).toEqual([ + { + rows: [ + { + sqft: 928.625, + sqm: 86.27243061926092, + }, + ], + fields: [ + { + name: 'sqft', + dataTypeID: 701, + }, + { + name: 'sqm', + dataTypeID: 701, + }, + ], + affectedRows: 0, + }, + ]) + + const area3 = await pg.exec(` + select ST_Area(geog) / 0.3048 ^ 2 sqft_spheroid, + ST_Area(geog, false) / 0.3048 ^ 2 sqft_sphere, + ST_Area(geog) sqm_spheroid + from ( + select ST_Transform( + 'SRID=2249;POLYGON((743238 2967416,743238 2967450,743265 2967450,743265.625 2967416,743238 2967416))'::geometry, + 4326 + ) :: geography geog + ) as subquery; + `) + + expect(area3).toEqual([ + { + rows: [ + { + sqft_spheroid: 928.6844047556697, + sqft_sphere: 926.609762750544, + sqm_spheroid: 86.27760440239217, + }, + ], + fields: [ + { + name: 'sqft_spheroid', + dataTypeID: 701, + }, + { + name: 'sqft_sphere', + dataTypeID: 701, + }, + { + name: 'sqm_spheroid', + dataTypeID: 701, + }, + ], + affectedRows: 0, + }, + ]) + }) + it('complex1', async () => { const pg = new PGlite({ extensions: { @@ -141,18 +256,6 @@ WHERE ST_Within(c.location, s.geom);`) ON ST_DWithin(c.geom::geography, r.geom::geography, 10000) ORDER BY distance_km; - -- Query: Compute buffered area around each river and intersecting cities - SELECT - r.name AS river_name, - COUNT(c.id) AS num_cities_intersecting, - ST_Area(ST_Transform(ST_Buffer(r.geom::geography, 5000), 3857)) / 1e6 AS buffer_area_sqkm - FROM rivers r - LEFT JOIN cities c - ON ST_Intersects(ST_Buffer(r.geom::geography, 5000), c.geom) - GROUP BY r.name; - - -- Cleanup test schema - -- DROP SCHEMA postgis_test CASCADE; `) }) }) From e4e2cefc946e091eaa3b849ce465f72f0c29a8ff Mon Sep 17 00:00:00 2001 From: tudor Date: Sat, 8 Nov 2025 10:01:20 +0100 Subject: [PATCH 59/79] topology test --- packages/pglite/tests/postgis.test.ts | 43 +++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/pglite/tests/postgis.test.ts b/packages/pglite/tests/postgis.test.ts index cd4a36236..5a7afb54c 100644 --- a/packages/pglite/tests/postgis.test.ts +++ b/packages/pglite/tests/postgis.test.ts @@ -194,6 +194,49 @@ WHERE ST_Within(c.location, s.geom);`) ]) }) + it('topology', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + defaultDataTransferContainer, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + const res = await pg.exec(` + WITH data(geom) AS (VALUES + ('LINESTRING (180 40, 30 20, 20 90)'::geometry) + ,('LINESTRING (180 40, 160 160)'::geometry) + ,('LINESTRING (80 60, 120 130, 150 80)'::geometry) + ,('LINESTRING (80 60, 150 80)'::geometry) + ,('LINESTRING (20 90, 70 70, 80 130)'::geometry) + ,('LINESTRING (80 130, 160 160)'::geometry) + ,('LINESTRING (20 90, 20 160, 70 190)'::geometry) + ,('LINESTRING (70 190, 80 130)'::geometry) + ,('LINESTRING (70 190, 160 160)'::geometry) + ) + SELECT ST_AsText( ST_Polygonize( geom )) + FROM data; + `) + + expect(res).toEqual([ + { + rows: [ + { + st_astext: + 'GEOMETRYCOLLECTION(POLYGON((180 40,30 20,20 90,70 70,80 130,160 160,180 40),(150 80,120 130,80 60,150 80)),POLYGON((80 60,120 130,150 80,80 60)),POLYGON((80 130,70 70,20 90,20 160,70 190,80 130)),POLYGON((160 160,80 130,70 190,160 160)))', + }, + ], + fields: [ + { + name: 'st_astext', + dataTypeID: 25, + }, + ], + affectedRows: 0, + }, + ]) + }) + it('complex1', async () => { const pg = new PGlite({ extensions: { From fbbc50c0523a1e4caec99f04e4f4e088de9934ad Mon Sep 17 00:00:00 2001 From: tudor Date: Sat, 8 Nov 2025 10:03:34 +0100 Subject: [PATCH 60/79] test desc --- packages/pglite/tests/postgis.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pglite/tests/postgis.test.ts b/packages/pglite/tests/postgis.test.ts index 5a7afb54c..b6ee6b36a 100644 --- a/packages/pglite/tests/postgis.test.ts +++ b/packages/pglite/tests/postgis.test.ts @@ -194,7 +194,7 @@ WHERE ST_Within(c.location, s.geom);`) ]) }) - it('topology', async () => { + it('ST_Polygonize', async () => { const pg = new PGlite({ extensions: { postgis, From 2a84e3812bd2ca73d4a4ac29ff4e700954e5e061 Mon Sep 17 00:00:00 2001 From: tudor Date: Sat, 8 Nov 2025 21:52:53 +0100 Subject: [PATCH 61/79] add postgis to REPL --- docs/repl/allExtensions.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/repl/allExtensions.ts b/docs/repl/allExtensions.ts index 1cfb1a674..3b266430d 100644 --- a/docs/repl/allExtensions.ts +++ b/docs/repl/allExtensions.ts @@ -24,6 +24,7 @@ export { pg_visibility } from '@electric-sql/pglite/contrib/pg_visibility' export { pg_walinspect } from '@electric-sql/pglite/contrib/pg_walinspect' export { pgtap } from '@electric-sql/pglite/pgtap' export { pg_uuidv7 } from '@electric-sql/pglite/pg_uuidv7' +export { postgis } from '@electric-sql/pglite/postgis' export { seg } from '@electric-sql/pglite/contrib/seg' export { tablefunc } from '@electric-sql/pglite/contrib/tablefunc' export { tcn } from '@electric-sql/pglite/contrib/tcn' From a2650e16d1a55f026614e8319b1ae0e201e8c212 Mon Sep 17 00:00:00 2001 From: tudor Date: Sun, 9 Nov 2025 09:32:18 +0100 Subject: [PATCH 62/79] postgis description in docs extensions --- docs/extensions/extensions.data.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/extensions/extensions.data.ts b/docs/extensions/extensions.data.ts index de0fde1c5..fa7d2fdd7 100644 --- a/docs/extensions/extensions.data.ts +++ b/docs/extensions/extensions.data.ts @@ -292,6 +292,19 @@ const baseExtensions: Extension[] = [ importName: 'pgtap', size: 239428, }, + { + name: 'postgis', + description: ` + PostGIS extends the capabilities of the PostgreSQL relational database by adding + support for storing, indexing, and querying geospatial data. + `, + shortDescription: 'Storing, indexing, and querying geospatial data.', + docs: 'postgis.net', + tags: ['postgres extension'], + importPath: '@electric-sql/pglite/postgis', + importName: 'postgis', + size: 7895452, + }, { name: 'pg_uuidv7', description: ` From de5ecbbed9d95bb26716d52e94fcbeb0775a4307 Mon Sep 17 00:00:00 2001 From: tudor Date: Sun, 9 Nov 2025 09:33:03 +0100 Subject: [PATCH 63/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 270db96a1..1984398b2 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 270db96a17e069951fb0bc64fbb409d9fefc3529 +Subproject commit 1984398b2759443e8a941ea0c0b860a6da5108ec From 0621fe64756a091162ba9bf42dafc337830e46f2 Mon Sep 17 00:00:00 2001 From: tudor Date: Sun, 9 Nov 2025 15:44:09 +0100 Subject: [PATCH 64/79] submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 1984398b2..fa3d6b6e4 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 1984398b2759443e8a941ea0c0b860a6da5108ec +Subproject commit fa3d6b6e46a645020c372dab59905fd40a242b89 From 8f4f9021a2e41a6363e3e5f30536f51ad5395ab1 Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 6 Jan 2026 11:41:58 +0100 Subject: [PATCH 65/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index b009f4a58..aceb78356 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit b009f4a58a09e5ce0f55f36b7d8edd570ae981df +Subproject commit aceb783568efdaaf6270f2dcd8b347f9e040c57f From ec2201c2e63838b7a1fe7f871a576833046df247 Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 6 Jan 2026 16:09:37 +0100 Subject: [PATCH 66/79] moved postgis extension to new npm package --- packages/pglite-postgis/.gitignore | 2 + packages/pglite-postgis/CHANGELOG.md | 7 + packages/pglite-postgis/README.md | 50 +++ packages/pglite-postgis/eslint.config.js | 22 ++ packages/pglite-postgis/package.json | 66 ++++ packages/pglite-postgis/src/index.ts | 17 ++ packages/pglite-postgis/tests/postgis.test.ts | 284 ++++++++++++++++++ packages/pglite-postgis/tsconfig.json | 8 + packages/pglite-postgis/tsup.config.ts | 26 ++ packages/pglite-postgis/vitest.config.ts | 10 + 10 files changed, 492 insertions(+) create mode 100644 packages/pglite-postgis/.gitignore create mode 100644 packages/pglite-postgis/CHANGELOG.md create mode 100644 packages/pglite-postgis/README.md create mode 100644 packages/pglite-postgis/eslint.config.js create mode 100644 packages/pglite-postgis/package.json create mode 100644 packages/pglite-postgis/src/index.ts create mode 100644 packages/pglite-postgis/tests/postgis.test.ts create mode 100644 packages/pglite-postgis/tsconfig.json create mode 100644 packages/pglite-postgis/tsup.config.ts create mode 100644 packages/pglite-postgis/vitest.config.ts diff --git a/packages/pglite-postgis/.gitignore b/packages/pglite-postgis/.gitignore new file mode 100644 index 000000000..ef8abb0e7 --- /dev/null +++ b/packages/pglite-postgis/.gitignore @@ -0,0 +1,2 @@ +release/* +dist \ No newline at end of file diff --git a/packages/pglite-postgis/CHANGELOG.md b/packages/pglite-postgis/CHANGELOG.md new file mode 100644 index 000000000..41b251d32 --- /dev/null +++ b/packages/pglite-postgis/CHANGELOG.md @@ -0,0 +1,7 @@ +# @electric-sql/pglite-postgis + +## 0.0.1 + +- Initial release +- PostGIS extension extracted from `@electric-sql/pglite` + diff --git a/packages/pglite-postgis/README.md b/packages/pglite-postgis/README.md new file mode 100644 index 000000000..1e671b7b3 --- /dev/null +++ b/packages/pglite-postgis/README.md @@ -0,0 +1,50 @@ +# @electric-sql/pglite-postgis + +PostGIS extension for [PGlite](https://pglite.dev). + +## Installation + +```bash +npm install @electric-sql/pglite-postgis +``` + +## Usage + +```typescript +import { PGlite } from '@electric-sql/pglite' +import { postgis } from '@electric-sql/pglite-postgis' + +const pg = new PGlite({ + extensions: { + postgis, + }, +}) + +await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + +// Create a table with geometry columns +await pg.exec(` + CREATE TABLE cities ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + location GEOMETRY(Point, 4326) + ); +`) + +// Insert data +await pg.query(` + INSERT INTO cities (name, location) + VALUES ('New York', ST_GeomFromText('POINT(-74.0060 40.7128)', 4326)) +`) + +// Query with spatial functions +const result = await pg.query(` + SELECT name, ST_AsText(location) as location + FROM cities +`) +``` + +## License + +Apache-2.0 + diff --git a/packages/pglite-postgis/eslint.config.js b/packages/pglite-postgis/eslint.config.js new file mode 100644 index 000000000..e001f1b83 --- /dev/null +++ b/packages/pglite-postgis/eslint.config.js @@ -0,0 +1,22 @@ +import globals from 'globals' +import rootConfig from '../../eslint.config.js' + +export default [ + ...rootConfig, + { + ignores: ['release/**/*', 'dist/**/*'], + }, + { + languageOptions: { + globals: { + ...globals.browser, + ...globals.node, + }, + }, + rules: { + ...rootConfig.rules, + '@typescript-eslint/no-explicit-any': 'off', + }, + }, +] + diff --git a/packages/pglite-postgis/package.json b/packages/pglite-postgis/package.json new file mode 100644 index 000000000..d5a768298 --- /dev/null +++ b/packages/pglite-postgis/package.json @@ -0,0 +1,66 @@ +{ + "name": "@electric-sql/pglite-postgis", + "version": "0.0.1", + "description": "PostGIS extension for PGlite", + "author": "Electric DB Limited", + "homepage": "https://pglite.dev", + "license": "Apache-2.0", + "keywords": [ + "postgres", + "sql", + "database", + "wasm", + "pglite", + "postgis", + "gis", + "geospatial" + ], + "private": false, + "publishConfig": { + "access": "public" + }, + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + } + }, + "files": [ + "dist" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/electric-sql/pglite.git", + "directory": "packages/pglite-postgis" + }, + "scripts": { + "build": "tsup", + "check:exports": "attw . --pack --profile node16", + "lint": "eslint ./src ./tests --report-unused-disable-directives --max-warnings 0", + "format": "prettier --write ./src ./tests", + "typecheck": "tsc", + "stylecheck": "pnpm lint && prettier --check ./src ./tests", + "test": "vitest", + "prepublishOnly": "pnpm check:exports" + }, + "devDependencies": { + "@arethetypeswrong/cli": "^0.18.1", + "@electric-sql/pglite": "workspace:*", + "@types/node": "^20.16.11", + "vitest": "^2.1.2" + }, + "peerDependencies": { + "@electric-sql/pglite": "workspace:0.3.14" + } +} + diff --git a/packages/pglite-postgis/src/index.ts b/packages/pglite-postgis/src/index.ts new file mode 100644 index 000000000..7638fbd62 --- /dev/null +++ b/packages/pglite-postgis/src/index.ts @@ -0,0 +1,17 @@ +import type { + Extension, + ExtensionSetupResult, + PGliteInterface, +} from '@electric-sql/pglite' + +const setup = async (_pg: PGliteInterface, emscriptenOpts: any) => { + return { + emscriptenOpts, + bundlePath: new URL('../release/postgis.tar.gz', import.meta.url), + } satisfies ExtensionSetupResult +} + +export const postgis = { + name: 'postgis', + setup, +} satisfies Extension diff --git a/packages/pglite-postgis/tests/postgis.test.ts b/packages/pglite-postgis/tests/postgis.test.ts new file mode 100644 index 000000000..c38ba765e --- /dev/null +++ b/packages/pglite-postgis/tests/postgis.test.ts @@ -0,0 +1,284 @@ +import { describe, it, expect } from 'vitest' +import { PGlite } from '@electric-sql/pglite' +import { postgis } from '../src/index.js' + +describe(`postgis`, () => { + it('basic', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + await pg.exec(` + CREATE TABLE vehicle_location ( + time TIMESTAMPTZ NOT NULL, + vehicle_id INT NOT NULL, + location GEOGRAPHY(POINT, 4326) +); + `) + const inserted = await pg.query(`INSERT INTO vehicle_location VALUES + ('2023-05-29 20:00:00', 1, 'POINT(15.3672 -87.7231)'), + ('2023-05-30 20:00:00', 1, 'POINT(15.3652 -80.7331)'), + ('2023-05-31 20:00:00', 1, 'POINT(15.2672 -85.7431)');`) + + expect(inserted.affectedRows).toEqual(3) + }), + it('cities', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + await pg.exec(` + CREATE TABLE cities ( + id SERIAL PRIMARY KEY, + name VARCHAR(100), + location GEOMETRY(Point, 4326) +); + `) + const inserted = await pg.query(`INSERT INTO cities (name, location) +VALUES + ('New York', ST_GeomFromText('POINT(-74.0060 40.7128)', 4326)), + ('Los Angeles', ST_GeomFromText('POINT(-118.2437 34.0522)', 4326)), + ('Chicago', ST_GeomFromText('POINT(-87.6298 41.8781)', 4326));`) + + expect(inserted.affectedRows).toEqual(3) + + const cities = await pg.query(`WITH state_boundary AS ( + SELECT ST_GeomFromText( + 'POLYGON((-91 36, -91 43, -87 43, -87 36, -91 36))', 4326 + ) AS geom +) +SELECT c.name +FROM cities c, state_boundary s +WHERE ST_Within(c.location, s.geom);`) + + expect(cities.affectedRows).toBe(0) + expect(cities.rows[0]).toEqual({ + name: 'Chicago', + }) + }) +}) + +it('areas', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + + const area1 = await pg.exec(` + select ST_Area(geom) sqft, + ST_Area(geom) * 0.3048 ^ 2 sqm + from ( + select 'SRID=2249;POLYGON((743238 2967416,743238 2967450, + 743265 2967450,743265.625 2967416,743238 2967416))' :: geometry geom + ) subquery;`) + + expect(area1).toEqual([ + { + rows: [ + { + sqft: 928.625, + sqm: 86.27208552, + }, + ], + fields: [ + { + name: 'sqft', + dataTypeID: 701, + }, + { + name: 'sqm', + dataTypeID: 701, + }, + ], + affectedRows: 0, + }, + ]) + + const area2 = await pg.exec(` + select ST_Area(geom) sqft, + ST_Area(ST_Transform(geom, 26986)) As sqm + from ( + select + 'SRID=2249;POLYGON((743238 2967416,743238 2967450, + 743265 2967450,743265.625 2967416,743238 2967416))' :: geometry geom + ) subquery; + + -- Cleanup test schema + -- DROP SCHEMA postgis_test CASCADE; + `) + + expect(area2).toEqual([ + { + rows: [ + { + sqft: 928.625, + sqm: 86.27243061926092, + }, + ], + fields: [ + { + name: 'sqft', + dataTypeID: 701, + }, + { + name: 'sqm', + dataTypeID: 701, + }, + ], + affectedRows: 0, + }, + ]) + + const area3 = await pg.exec(` + select ST_Area(geog) / 0.3048 ^ 2 sqft_spheroid, + ST_Area(geog, false) / 0.3048 ^ 2 sqft_sphere, + ST_Area(geog) sqm_spheroid + from ( + select ST_Transform( + 'SRID=2249;POLYGON((743238 2967416,743238 2967450,743265 2967450,743265.625 2967416,743238 2967416))'::geometry, + 4326 + ) :: geography geog + ) as subquery; + `) + + expect(area3).toEqual([ + { + rows: [ + { + sqft_spheroid: 928.6844047556697, + sqft_sphere: 926.609762750544, + sqm_spheroid: 86.27760440239217, + }, + ], + fields: [ + { + name: 'sqft_spheroid', + dataTypeID: 701, + }, + { + name: 'sqft_sphere', + dataTypeID: 701, + }, + { + name: 'sqm_spheroid', + dataTypeID: 701, + }, + ], + affectedRows: 0, + }, + ]) +}) + +it('ST_Polygonize', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + const res = await pg.exec(` + WITH data(geom) AS (VALUES + ('LINESTRING (180 40, 30 20, 20 90)'::geometry) + ,('LINESTRING (180 40, 160 160)'::geometry) + ,('LINESTRING (80 60, 120 130, 150 80)'::geometry) + ,('LINESTRING (80 60, 150 80)'::geometry) + ,('LINESTRING (20 90, 70 70, 80 130)'::geometry) + ,('LINESTRING (80 130, 160 160)'::geometry) + ,('LINESTRING (20 90, 20 160, 70 190)'::geometry) + ,('LINESTRING (70 190, 80 130)'::geometry) + ,('LINESTRING (70 190, 160 160)'::geometry) + ) + SELECT ST_AsText( ST_Polygonize( geom )) + FROM data; + `) + + expect(res).toEqual([ + { + rows: [ + { + st_astext: + 'GEOMETRYCOLLECTION(POLYGON((180 40,30 20,20 90,70 70,80 130,160 160,180 40),(150 80,120 130,80 60,150 80)),POLYGON((80 60,120 130,150 80,80 60)),POLYGON((80 130,70 70,20 90,20 160,70 190,80 130)),POLYGON((160 160,80 130,70 190,160 160)))', + }, + ], + fields: [ + { + name: 'st_astext', + dataTypeID: 25, + }, + ], + affectedRows: 0, + }, + ]) +}) + +it('complex1', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + + await pg.exec(` + -- Create test schema + -- CREATE SCHEMA IF NOT EXISTS postgis_test; + -- SET search_path TO postgis_test; + + -- Create a table with geometry columns + CREATE TABLE cities ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + population INTEGER, + geom GEOMETRY(Point, 4326) + );`) + + await pg.exec(` + CREATE TABLE rivers ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + geom GEOMETRY(LineString, 4326) + ); + + -- Insert sample data + INSERT INTO cities (name, population, geom) VALUES + ('Paris', 2148000, ST_SetSRID(ST_MakePoint(2.3522, 48.8566), 4326)), + ('Berlin', 3769000, ST_SetSRID(ST_MakePoint(13.4050, 52.5200), 4326)), + ('London', 8982000, ST_SetSRID(ST_MakePoint(-0.1276, 51.5072), 4326)), + ('Amsterdam', 872757, ST_SetSRID(ST_MakePoint(4.9041, 52.3676), 4326)); + + INSERT INTO rivers (name, geom) VALUES + ('Seine', ST_SetSRID(ST_MakeLine(ARRAY[ + ST_MakePoint(2.1, 48.8), + ST_MakePoint(2.35, 48.85), + ST_MakePoint(2.45, 48.9) + ]), 4326)), + ('Spree', ST_SetSRID(ST_MakeLine(ARRAY[ + ST_MakePoint(13.1, 52.4), + ST_MakePoint(13.35, 52.5), + ST_MakePoint(13.45, 52.52) + ]), 4326)); + + -- Create spatial index + CREATE INDEX idx_cities_geom ON cities USING GIST (geom); + CREATE INDEX idx_rivers_geom ON rivers USING GIST (geom); + + -- Query: Find cities within 10 km of any river + SELECT + c.name AS city, + r.name AS river, + ST_Distance(c.geom::geography, r.geom::geography) AS distance_km + FROM cities c + JOIN rivers r + ON ST_DWithin(c.geom::geography, r.geom::geography, 10000) + ORDER BY distance_km; + + `) +}) diff --git a/packages/pglite-postgis/tsconfig.json b/packages/pglite-postgis/tsconfig.json new file mode 100644 index 000000000..6aab239b1 --- /dev/null +++ b/packages/pglite-postgis/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["src", "tsup.config.ts", "vitest.config.ts"] +} + diff --git a/packages/pglite-postgis/tsup.config.ts b/packages/pglite-postgis/tsup.config.ts new file mode 100644 index 000000000..0020b9fe5 --- /dev/null +++ b/packages/pglite-postgis/tsup.config.ts @@ -0,0 +1,26 @@ +import { cpSync } from 'fs' +import { resolve } from 'path' +import { defineConfig } from 'tsup' + +const entryPoints = ['src/index.ts'] + +const minify = process.env.DEBUG === 'true' ? false : true + +export default defineConfig([ + { + entry: entryPoints, + sourcemap: true, + dts: { + entry: entryPoints, + resolve: true, + }, + clean: true, + minify: minify, + shims: true, + format: ['esm', 'cjs'], + onSuccess: async () => { + cpSync(resolve('release/postgis.tar.gz'), resolve('dist/postgis.tar.gz')) + }, + }, +]) + diff --git a/packages/pglite-postgis/vitest.config.ts b/packages/pglite-postgis/vitest.config.ts new file mode 100644 index 000000000..3144cb036 --- /dev/null +++ b/packages/pglite-postgis/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + testTimeout: 30000, + }, +}) + From 548c2c9b8907ed649ee7205e333b5b5d31bc55d5 Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 6 Jan 2026 16:10:16 +0100 Subject: [PATCH 67/79] moved postgis extension to new npm package 2 --- docs/extensions/extensions.data.ts | 5 +- docs/package.json | 1 + docs/repl/allExtensions.ts | 2 +- package.json | 3 +- packages/pglite/package.json | 10 - packages/pglite/src/postgis/index.ts | 17 -- packages/pglite/tests/postgis.test.ts | 304 -------------------------- packages/pglite/tsup.config.ts | 1 - pnpm-lock.yaml | 19 +- 9 files changed, 25 insertions(+), 337 deletions(-) delete mode 100644 packages/pglite/src/postgis/index.ts delete mode 100644 packages/pglite/tests/postgis.test.ts diff --git a/docs/extensions/extensions.data.ts b/docs/extensions/extensions.data.ts index bb2c5be87..d781c73ed 100644 --- a/docs/extensions/extensions.data.ts +++ b/docs/extensions/extensions.data.ts @@ -297,13 +297,14 @@ const baseExtensions: Extension[] = [ description: ` PostGIS extends the capabilities of the PostgreSQL relational database by adding support for storing, indexing, and querying geospatial data. + *No GDAL support atm. `, shortDescription: 'Storing, indexing, and querying geospatial data.', docs: 'postgis.net', tags: ['postgres extension'], - importPath: '@electric-sql/pglite/postgis', + importPath: '@electric-sql/pglite-postgis', importName: 'postgis', - size: 7895452, + size: 7901736, }, { name: 'pg_uuidv7', diff --git a/docs/package.json b/docs/package.json index 03864e194..95ad3ef61 100644 --- a/docs/package.json +++ b/docs/package.json @@ -20,6 +20,7 @@ "dependencies": { "@electric-sql/pglite": "workspace:*", "@electric-sql/pglite-repl": "workspace:*", + "@electric-sql/pglite-postgis": "workspace:*", "@uiw/codemirror-theme-github": "^4.23.0", "dedent": "^1.5.3" } diff --git a/docs/repl/allExtensions.ts b/docs/repl/allExtensions.ts index 1bc76b2be..91489ef34 100644 --- a/docs/repl/allExtensions.ts +++ b/docs/repl/allExtensions.ts @@ -24,7 +24,7 @@ export { pg_visibility } from '@electric-sql/pglite/contrib/pg_visibility' export { pg_walinspect } from '@electric-sql/pglite/contrib/pg_walinspect' export { pgtap } from '@electric-sql/pglite/pgtap' export { pg_uuidv7 } from '@electric-sql/pglite/pg_uuidv7' -export { postgis } from '@electric-sql/pglite/postgis' +export { postgis } from '@electric-sql/pglite-postgis' export { seg } from '@electric-sql/pglite/contrib/seg' export { tablefunc } from '@electric-sql/pglite/contrib/tablefunc' export { tcn } from '@electric-sql/pglite/contrib/tcn' diff --git a/package.json b/package.json index 600dd8143..d983ff93b 100644 --- a/package.json +++ b/package.json @@ -12,9 +12,10 @@ "ci:publish": "pnpm changeset publish", "ts:build": "pnpm -r --filter \"./packages/**\" build", "ts:build:debug": "DEBUG=true pnpm ts:build", + "wasm:copy-postgis": "mkdir -p ./packages/pglite-postgis/release && cp ./postgres-pglite/dist/extensions/postgis/postgis.tar.gz ./packages/pglite-postgis/release", "wasm:copy-pgdump": "mkdir -p ./packages/pglite-tools/release && cp ./postgres-pglite/dist/bin/pg_dump.* ./packages/pglite-tools/release", "wasm:copy-pglite": "mkdir -p ./packages/pglite/release/ && cp ./postgres-pglite/dist/bin/pglite.* ./packages/pglite/release/ && cp ./postgres-pglite/dist/extensions/*.tar.gz ./packages/pglite/release/", - "wasm:build": "cd postgres-pglite && ./build-with-docker.sh && cd .. && pnpm wasm:copy-pglite && pnpm wasm:copy-pgdump", + "wasm:build": "cd postgres-pglite && ./build-with-docker.sh && cd .. && pnpm wasm:copy-pglite && pnpm wasm:copy-pgdump && pnpm wasm:copy-postgis", "wasm:build:debug": "DEBUG=true pnpm wasm:build", "build:all": "pnpm wasm:build && pnpm ts:build", "build:all:debug": "DEBUG=true pnpm build:all" diff --git a/packages/pglite/package.json b/packages/pglite/package.json index dd5d91a93..9a85b0e10 100644 --- a/packages/pglite/package.json +++ b/packages/pglite/package.json @@ -80,16 +80,6 @@ "default": "./dist/pg_ivm/index.cjs" } }, - "./postgis": { - "import": { - "types": "./dist/postgis/index.d.ts", - "default": "./dist/postgis/index.js" - }, - "require": { - "types": "./dist/postgis/index.d.cts", - "default": "./dist/postgis/index.cjs" - } - }, "./pgtap": { "import": { "types": "./dist/pgtap/index.d.ts", diff --git a/packages/pglite/src/postgis/index.ts b/packages/pglite/src/postgis/index.ts deleted file mode 100644 index 9e4e37d95..000000000 --- a/packages/pglite/src/postgis/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { - Extension, - ExtensionSetupResult, - PGliteInterface, -} from '../interface' - -const setup = async (_pg: PGliteInterface, emscriptenOpts: any) => { - return { - emscriptenOpts, - bundlePath: new URL('../../release/postgis.tar.gz', import.meta.url), - } satisfies ExtensionSetupResult -} - -export const postgis = { - name: 'postgis', - setup, -} satisfies Extension diff --git a/packages/pglite/tests/postgis.test.ts b/packages/pglite/tests/postgis.test.ts deleted file mode 100644 index b6ee6b36a..000000000 --- a/packages/pglite/tests/postgis.test.ts +++ /dev/null @@ -1,304 +0,0 @@ -import { describe, it, expect } from 'vitest' -import { testEsmCjsAndDTC } from './test-utils.ts' - -await testEsmCjsAndDTC(async (importType, defaultDataTransferContainer) => { - const { PGlite } = - importType === 'esm' - ? await import('../dist/index.js') - : ((await import( - '../dist/index.cjs' - )) as unknown as typeof import('../dist/index.js')) - - const { postgis } = - importType === 'esm' - ? await import('../dist/postgis/index.js') - : ((await import( - '../dist/postgis/index.cjs' - )) as unknown as typeof import('../dist/postgis/index.js')) - - describe(`postgis`, () => { - it('basic', async () => { - const pg = new PGlite({ - extensions: { - postgis, - }, - defaultDataTransferContainer, - }) - - await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') - await pg.exec(` - CREATE TABLE vehicle_location ( - time TIMESTAMPTZ NOT NULL, - vehicle_id INT NOT NULL, - location GEOGRAPHY(POINT, 4326) -); - `) - const inserted = await pg.query(`INSERT INTO vehicle_location VALUES - ('2023-05-29 20:00:00', 1, 'POINT(15.3672 -87.7231)'), - ('2023-05-30 20:00:00', 1, 'POINT(15.3652 -80.7331)'), - ('2023-05-31 20:00:00', 1, 'POINT(15.2672 -85.7431)');`) - - expect(inserted.affectedRows).toEqual(3) - }), - it('cities', async () => { - const pg = new PGlite({ - extensions: { - postgis, - }, - defaultDataTransferContainer, - }) - - await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') - await pg.exec(` - CREATE TABLE cities ( - id SERIAL PRIMARY KEY, - name VARCHAR(100), - location GEOMETRY(Point, 4326) -); - `) - const inserted = await pg.query(`INSERT INTO cities (name, location) -VALUES - ('New York', ST_GeomFromText('POINT(-74.0060 40.7128)', 4326)), - ('Los Angeles', ST_GeomFromText('POINT(-118.2437 34.0522)', 4326)), - ('Chicago', ST_GeomFromText('POINT(-87.6298 41.8781)', 4326));`) - - expect(inserted.affectedRows).toEqual(3) - - const cities = await pg.query(`WITH state_boundary AS ( - SELECT ST_GeomFromText( - 'POLYGON((-91 36, -91 43, -87 43, -87 36, -91 36))', 4326 - ) AS geom -) -SELECT c.name -FROM cities c, state_boundary s -WHERE ST_Within(c.location, s.geom);`) - - expect(cities.affectedRows).toBe(0) - expect(cities.rows[0]).toEqual({ - name: 'Chicago', - }) - }) - }) - - it('areas', async () => { - const pg = new PGlite({ - extensions: { - postgis, - }, - defaultDataTransferContainer, - }) - await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') - - const area1 = await pg.exec(` - select ST_Area(geom) sqft, - ST_Area(geom) * 0.3048 ^ 2 sqm - from ( - select 'SRID=2249;POLYGON((743238 2967416,743238 2967450, - 743265 2967450,743265.625 2967416,743238 2967416))' :: geometry geom - ) subquery;`) - - expect(area1).toEqual([ - { - rows: [ - { - sqft: 928.625, - sqm: 86.27208552, - }, - ], - fields: [ - { - name: 'sqft', - dataTypeID: 701, - }, - { - name: 'sqm', - dataTypeID: 701, - }, - ], - affectedRows: 0, - }, - ]) - - const area2 = await pg.exec(` - select ST_Area(geom) sqft, - ST_Area(ST_Transform(geom, 26986)) As sqm - from ( - select - 'SRID=2249;POLYGON((743238 2967416,743238 2967450, - 743265 2967450,743265.625 2967416,743238 2967416))' :: geometry geom - ) subquery; - - -- Cleanup test schema - -- DROP SCHEMA postgis_test CASCADE; - `) - - expect(area2).toEqual([ - { - rows: [ - { - sqft: 928.625, - sqm: 86.27243061926092, - }, - ], - fields: [ - { - name: 'sqft', - dataTypeID: 701, - }, - { - name: 'sqm', - dataTypeID: 701, - }, - ], - affectedRows: 0, - }, - ]) - - const area3 = await pg.exec(` - select ST_Area(geog) / 0.3048 ^ 2 sqft_spheroid, - ST_Area(geog, false) / 0.3048 ^ 2 sqft_sphere, - ST_Area(geog) sqm_spheroid - from ( - select ST_Transform( - 'SRID=2249;POLYGON((743238 2967416,743238 2967450,743265 2967450,743265.625 2967416,743238 2967416))'::geometry, - 4326 - ) :: geography geog - ) as subquery; - `) - - expect(area3).toEqual([ - { - rows: [ - { - sqft_spheroid: 928.6844047556697, - sqft_sphere: 926.609762750544, - sqm_spheroid: 86.27760440239217, - }, - ], - fields: [ - { - name: 'sqft_spheroid', - dataTypeID: 701, - }, - { - name: 'sqft_sphere', - dataTypeID: 701, - }, - { - name: 'sqm_spheroid', - dataTypeID: 701, - }, - ], - affectedRows: 0, - }, - ]) - }) - - it('ST_Polygonize', async () => { - const pg = new PGlite({ - extensions: { - postgis, - }, - defaultDataTransferContainer, - }) - await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') - const res = await pg.exec(` - WITH data(geom) AS (VALUES - ('LINESTRING (180 40, 30 20, 20 90)'::geometry) - ,('LINESTRING (180 40, 160 160)'::geometry) - ,('LINESTRING (80 60, 120 130, 150 80)'::geometry) - ,('LINESTRING (80 60, 150 80)'::geometry) - ,('LINESTRING (20 90, 70 70, 80 130)'::geometry) - ,('LINESTRING (80 130, 160 160)'::geometry) - ,('LINESTRING (20 90, 20 160, 70 190)'::geometry) - ,('LINESTRING (70 190, 80 130)'::geometry) - ,('LINESTRING (70 190, 160 160)'::geometry) - ) - SELECT ST_AsText( ST_Polygonize( geom )) - FROM data; - `) - - expect(res).toEqual([ - { - rows: [ - { - st_astext: - 'GEOMETRYCOLLECTION(POLYGON((180 40,30 20,20 90,70 70,80 130,160 160,180 40),(150 80,120 130,80 60,150 80)),POLYGON((80 60,120 130,150 80,80 60)),POLYGON((80 130,70 70,20 90,20 160,70 190,80 130)),POLYGON((160 160,80 130,70 190,160 160)))', - }, - ], - fields: [ - { - name: 'st_astext', - dataTypeID: 25, - }, - ], - affectedRows: 0, - }, - ]) - }) - - it('complex1', async () => { - const pg = new PGlite({ - extensions: { - postgis, - }, - defaultDataTransferContainer, - }) - await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') - - await pg.exec(` - -- Create test schema - -- CREATE SCHEMA IF NOT EXISTS postgis_test; - -- SET search_path TO postgis_test; - - -- Create a table with geometry columns - CREATE TABLE cities ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - population INTEGER, - geom GEOMETRY(Point, 4326) - );`) - - await pg.exec(` - CREATE TABLE rivers ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - geom GEOMETRY(LineString, 4326) - ); - - -- Insert sample data - INSERT INTO cities (name, population, geom) VALUES - ('Paris', 2148000, ST_SetSRID(ST_MakePoint(2.3522, 48.8566), 4326)), - ('Berlin', 3769000, ST_SetSRID(ST_MakePoint(13.4050, 52.5200), 4326)), - ('London', 8982000, ST_SetSRID(ST_MakePoint(-0.1276, 51.5072), 4326)), - ('Amsterdam', 872757, ST_SetSRID(ST_MakePoint(4.9041, 52.3676), 4326)); - - INSERT INTO rivers (name, geom) VALUES - ('Seine', ST_SetSRID(ST_MakeLine(ARRAY[ - ST_MakePoint(2.1, 48.8), - ST_MakePoint(2.35, 48.85), - ST_MakePoint(2.45, 48.9) - ]), 4326)), - ('Spree', ST_SetSRID(ST_MakeLine(ARRAY[ - ST_MakePoint(13.1, 52.4), - ST_MakePoint(13.35, 52.5), - ST_MakePoint(13.45, 52.52) - ]), 4326)); - - -- Create spatial index - CREATE INDEX idx_cities_geom ON cities USING GIST (geom); - CREATE INDEX idx_rivers_geom ON rivers USING GIST (geom); - - -- Query: Find cities within 10 km of any river - SELECT - c.name AS city, - r.name AS river, - ST_Distance(c.geom::geography, r.geom::geography) AS distance_km - FROM cities c - JOIN rivers r - ON ST_DWithin(c.geom::geography, r.geom::geography, 10000) - ORDER BY distance_km; - - `) - }) -}) diff --git a/packages/pglite/tsup.config.ts b/packages/pglite/tsup.config.ts index a0518b1ba..a73508d8b 100644 --- a/packages/pglite/tsup.config.ts +++ b/packages/pglite/tsup.config.ts @@ -25,7 +25,6 @@ const entryPoints = [ 'src/live/index.ts', 'src/vector/index.ts', 'src/pg_ivm/index.ts', - 'src/postgis/index.ts', 'src/pgtap/index.ts', 'src/pg_uuidv7/index.ts', 'src/worker/index.ts', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4fe1e8540..17ca10cf3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,6 +44,9 @@ importers: '@electric-sql/pglite': specifier: workspace:* version: link:../packages/pglite + '@electric-sql/pglite-postgis': + specifier: workspace:* + version: link:../packages/pglite-postgis '@electric-sql/pglite-repl': specifier: workspace:* version: link:../packages/pglite-repl @@ -199,6 +202,21 @@ importers: specifier: ^2.1.2 version: 2.1.2(@types/node@20.16.11)(jsdom@24.1.3)(terser@5.34.1) + packages/pglite-postgis: + devDependencies: + '@arethetypeswrong/cli': + specifier: ^0.18.1 + version: 0.18.1 + '@electric-sql/pglite': + specifier: workspace:* + version: link:../pglite + '@types/node': + specifier: ^20.16.11 + version: 20.16.11 + vitest: + specifier: ^2.1.2 + version: 2.1.2(@types/node@20.16.11)(jsdom@24.1.3)(terser@5.34.1) + packages/pglite-react: devDependencies: '@arethetypeswrong/cli': @@ -2074,7 +2092,6 @@ packages: bun@1.1.30: resolution: {integrity: sha512-ysRL1pq10Xba0jqVLPrKU3YIv0ohfp3cTajCPtpjCyppbn3lfiAVNpGoHfyaxS17OlPmWmR67UZRPw/EueQuug==} - cpu: [arm64, x64] os: [darwin, linux, win32] hasBin: true From dcbea3006aff57be3ad376ad1c2eab6e39243812 Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 6 Jan 2026 16:10:26 +0100 Subject: [PATCH 68/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index aceb78356..b798740ae 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit aceb783568efdaaf6270f2dcd8b347f9e040c57f +Subproject commit b798740aed34a3acac1b706a1459b1a35cff203e From 2e128161f779d19846e26dce526cc2f25b1378d1 Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 6 Jan 2026 17:11:11 +0100 Subject: [PATCH 69/79] udpate submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index b798740ae..7d256dcf1 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit b798740aed34a3acac1b706a1459b1a35cff203e +Subproject commit 7d256dcf1e3b9c95e6545586bb9291c29c0c27db From f73d6594c1cc174af338a081423561cc73a5ada3 Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 6 Jan 2026 20:43:41 +0100 Subject: [PATCH 70/79] update CI --- .github/workflows/build_and_test.yml | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3a18acca4..50b8aee7a 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -64,10 +64,15 @@ jobs: - name: Typecheck pglite working-directory: ${{ github.workspace }}/packages/pglite run: pnpm typecheck + - name: Test pglite working-directory: ${{ github.workspace }}/packages/pglite run: pnpm test + - name: Test pglite-postgis + working-directory: ${{ github.workspace }}/packages/pglite-postgis + run: pnpm test + - name: Upload PGlite Interim to Github artifacts id: upload-pglite-interim-build-files uses: actions/upload-artifact@v4 @@ -84,6 +89,14 @@ jobs: path: ./packages/pglite-tools/release/** retention-days: 60 + - name: Upload pglite-postgis build artifacts to Github artifacts + id: upload-pglite-postgis-release-files + uses: actions/upload-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release/** + retention-days: 60 + build-and-test-pglite: name: Build and Test packages/pglite runs-on: blacksmith-32vcpu-ubuntu-2204 @@ -118,6 +131,12 @@ jobs: name: pglite-tools-release-files-node-v20.x path: ./packages/pglite-tools/release + - name: Download pglite-postgis build artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release + - name: Install dependencies run: | pnpm install --frozen-lockfile @@ -203,6 +222,12 @@ jobs: with: name: pglite-tools-release-files-node-v20.x path: ./packages/pglite-tools/release/ + + - name: Download pglite-postgis build artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release/ - name: Install dependencies run: pnpm install --frozen-lockfile @@ -253,6 +278,12 @@ jobs: name: pglite-tools-release-files-node-v20.x path: ./packages/pglite-tools/release + - name: Download pglite-postgis build artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release/ + - name: Download PGlite build artifacts uses: actions/download-artifact@v4 with: @@ -366,6 +397,12 @@ jobs: name: pglite-tools-release-files-node-v20.x path: ./packages/pglite-tools/release/ + - name: Download pglite-postgis build artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release/ + - run: pnpm install --frozen-lockfile - run: pnpm --filter "./packages/**" build - name: Create Release Pull Request or Publish From a8d8f200fd6b5ed8abf7167a634d0ac960dff3eb Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 6 Jan 2026 21:36:19 +0100 Subject: [PATCH 71/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index c29691b0b..670a4bfb4 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit c29691b0b1830d1b723f2704cd3e8f38385cf12a +Subproject commit 670a4bfb42d5402fa129fccf511cdb8b12125205 From 647c87ea4894a809dbfa9cc2e718b315d7b5011f Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 6 Jan 2026 22:03:09 +0100 Subject: [PATCH 72/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 670a4bfb4..165e39b0c 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 670a4bfb42d5402fa129fccf511cdb8b12125205 +Subproject commit 165e39b0c0b871bc7c019df3cff95e9b0c7f533b From 14bff079e6f6124714ca3aba9bdcc1bea5111931 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 7 Jan 2026 09:27:05 +0100 Subject: [PATCH 73/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 165e39b0c..a47386c1d 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 165e39b0c0b871bc7c019df3cff95e9b0c7f533b +Subproject commit a47386c1d6742f1120eb27ebbe3b8af2f55aa3f1 From 6162e78ddececcb99e8c08ac70a726945abdde56 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 7 Jan 2026 09:50:23 +0100 Subject: [PATCH 74/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index a47386c1d..8264bb661 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit a47386c1d6742f1120eb27ebbe3b8af2f55aa3f1 +Subproject commit 8264bb661bb2e385b4172641f0f7350584843707 From bd855839155322951bf5f0f3a18bdaac239a06f8 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 7 Jan 2026 10:37:58 +0100 Subject: [PATCH 75/79] update workflow --- .github/workflows/build_and_test.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 50b8aee7a..9dd408125 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -249,7 +249,14 @@ jobs: uses: actions/upload-artifact@v4 with: name: pglite-tools-dist-node-v${{ matrix.node }} - path: ./packages/pglite-tools/dist/* + path: ./packages/pglite-tools/dist/* + + - name: Upload pglite-postgis distribution artifact + id: upload-pglite-postgis-dist + uses: actions/upload-artifact@v4 + with: + name: pglite-postgis-dist-node-v${{ matrix.node }} + path: ./packages/pglite-postgis/dist/* publish-website-with-demos: name: Publish website with demos @@ -290,6 +297,12 @@ jobs: name: pglite-dist-node-v20.x path: ./packages/pglite/dist/ + - name: Download pglite-postgis dist artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-dist-node-v20.x + path: ./packages/pglite-postgis/dist/ + - name: Install dependencies run: pnpm install --frozen-lockfile From 7d863a6e80fd80d4d0e218ece1ed59328ff2c6e2 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 7 Jan 2026 10:38:02 +0100 Subject: [PATCH 76/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 8264bb661..96993ac54 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 8264bb661bb2e385b4172641f0f7350584843707 +Subproject commit 96993ac54ff2a5d97015d0e839ef87eeab2c5c63 From 47abdb12af8784e33e5c84da73fc267f39b2a90d Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 7 Jan 2026 10:44:21 +0100 Subject: [PATCH 77/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 96993ac54..17bc8bf6e 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 96993ac54ff2a5d97015d0e839ef87eeab2c5c63 +Subproject commit 17bc8bf6ef03a6eda8704281fef386396d555693 From 4c27eb6f7bd0998bc16e94a3bd37ef87541a1a6a Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 7 Jan 2026 12:33:24 +0100 Subject: [PATCH 78/79] allow extensions to be loaded in pglite-socket --- packages/pglite-socket/package.json | 4 +- packages/pglite-socket/src/scripts/server.ts | 70 +++++++++++++++++++- pnpm-lock.yaml | 3 + 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/packages/pglite-socket/package.json b/packages/pglite-socket/package.json index c5c6cf21e..eb9da0b88 100644 --- a/packages/pglite-socket/package.json +++ b/packages/pglite-socket/package.json @@ -52,6 +52,7 @@ "@arethetypeswrong/cli": "^0.18.1", "@electric-sql/pg-protocol": "workspace:*", "@electric-sql/pglite": "workspace:*", + "@electric-sql/pglite-postgis": "workspace:*", "@types/emscripten": "^1.41.1", "@types/node": "^20.16.11", "pg": "^8.14.0", @@ -60,6 +61,7 @@ "vitest": "^1.3.1" }, "peerDependencies": { - "@electric-sql/pglite": "workspace:*" + "@electric-sql/pglite": "workspace:*", + "@electric-sql/pglite-postgis": "workspace:*" } } diff --git a/packages/pglite-socket/src/scripts/server.ts b/packages/pglite-socket/src/scripts/server.ts index 552295625..be4465f86 100644 --- a/packages/pglite-socket/src/scripts/server.ts +++ b/packages/pglite-socket/src/scripts/server.ts @@ -1,6 +1,7 @@ #!/usr/bin/env node import { PGlite, DebugLevel } from '@electric-sql/pglite' +import type { Extension, Extensions } from '@electric-sql/pglite' import { PGLiteSocketServer } from '../index' import { parseArgs } from 'node:util' import { spawn, ChildProcess } from 'node:child_process' @@ -38,6 +39,12 @@ const args = parseArgs({ default: '0', help: 'Debug level (0-5)', }, + extensions: { + type: 'string', + short: 'e', + default: undefined, + help: 'Comma-separated list of extensions to load (e.g., vector,pgcrypto,postgis)', + }, run: { type: 'string', short: 'r', @@ -72,6 +79,7 @@ Options: -h, --host=HOST Host to bind to (default: 127.0.0.1) -u, --path=UNIX Unix socket to bind to (default: undefined). Takes precedence over host:port -v, --debug=LEVEL Debug level 0-5 (default: 0) + -e, --extensions=LIST Comma-separated list of extensions to load (e.g., vector,pgcrypto,postgis) -r, --run=COMMAND Command to run after server starts --include-database-url Include DATABASE_URL in subprocess environment --shutdown-timeout=MS Timeout for graceful subprocess shutdown in ms (default: 5000) @@ -83,6 +91,7 @@ interface ServerConfig { host: string path?: string debugLevel: DebugLevel + extensionNames?: string[] runCommand?: string includeDatabaseUrl: boolean shutdownTimeout: number @@ -99,12 +108,16 @@ class PGLiteServerRunner { } static parseConfig(): ServerConfig { + const extensionsArg = args.values.extensions as string | undefined return { dbPath: args.values.db as string, port: parseInt(args.values.port as string, 10), host: args.values.host as string, path: args.values.path as string, debugLevel: parseInt(args.values.debug as string, 10) as DebugLevel, + extensionNames: extensionsArg + ? extensionsArg.split(',').map((e) => e.trim()) + : undefined, runCommand: args.values.run as string, includeDatabaseUrl: args.values['include-database-url'] as boolean, shutdownTimeout: parseInt(args.values['shutdown-timeout'] as string, 10), @@ -126,11 +139,66 @@ class PGLiteServerRunner { } } + private async importExtensions(): Promise { + if (!this.config.extensionNames?.length) { + return undefined + } + + const extensions: Extensions = {} + + // Built-in extensions that are not in contrib + const builtInExtensions = [ + 'vector', + 'live', + 'pg_hashids', + 'pg_ivm', + 'pg_uuidv7', + 'pgtap', + ] + + for (const name of this.config.extensionNames) { + let ext: Extension | null = null + + try { + if (builtInExtensions.includes(name)) { + // Built-in extension (e.g., @electric-sql/pglite/vector) + const mod = await import(`@electric-sql/pglite/${name}`) + ext = mod[name] as Extension + } else { + // Try contrib first (e.g., @electric-sql/pglite/contrib/pgcrypto) + try { + const mod = await import(`@electric-sql/pglite/contrib/${name}`) + ext = mod[name] as Extension + } catch { + // Fall back to external package (e.g., @electric-sql/pglite-postgis) + const mod = await import(`@electric-sql/pglite-${name}`) + ext = mod[name] as Extension + } + } + + if (ext) { + extensions[name] = ext + console.log(`Imported extension: ${name}`) + } + } catch (error) { + console.error(`Failed to import extension '${name}':`, error) + throw new Error(`Failed to import extension '${name}'`) + } + } + + return Object.keys(extensions).length > 0 ? extensions : undefined + } + private async initializeDatabase(): Promise { console.log(`Initializing PGLite with database: ${this.config.dbPath}`) console.log(`Debug level: ${this.config.debugLevel}`) - this.db = new PGlite(this.config.dbPath, { debug: this.config.debugLevel }) + const extensions = await this.importExtensions() + + this.db = new PGlite(this.config.dbPath, { + debug: this.config.debugLevel, + extensions, + }) await this.db.waitReady console.log('PGlite database initialized') } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bba6cdff2..162c10daa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -358,6 +358,9 @@ importers: '@electric-sql/pglite': specifier: workspace:* version: link:../pglite + '@electric-sql/pglite-postgis': + specifier: workspace:* + version: link:../pglite-postgis '@types/emscripten': specifier: ^1.41.1 version: 1.41.1 From c9b801028850c6cb8cc6876d0c49af23a0f164c3 Mon Sep 17 00:00:00 2001 From: tudor Date: Thu, 8 Jan 2026 09:30:30 +0100 Subject: [PATCH 79/79] update submodule --- postgres-pglite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-pglite b/postgres-pglite index 17bc8bf6e..fb2c2f16c 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 17bc8bf6ef03a6eda8704281fef386396d555693 +Subproject commit fb2c2f16c6a44c9d1c380e24f14d24ff76bac9d5