From 805bb611b4382a4e17709b2125d0ed14e926cff2 Mon Sep 17 00:00:00 2001 From: Marcelo Lv Cabral Date: Fri, 19 Dec 2025 11:37:11 -0700 Subject: [PATCH 1/2] Add support for synchronous backend configuration --- documentation/configuration.md | 18 +++++ src/backends/cow.ts | 7 +- src/backends/store/fs.ts | 80 ++++++++++++++++++ src/config.ts | 143 ++++++++++++++++++++++++++++++++- src/internal/filesystem.ts | 14 ++++ src/mixins/mutexed.ts | 5 ++ tests/common/config.test.ts | 66 +++++++++++++++ 7 files changed, 331 insertions(+), 2 deletions(-) create mode 100644 tests/common/config.test.ts diff --git a/documentation/configuration.md b/documentation/configuration.md index 1cb99e91..8eceab50 100644 --- a/documentation/configuration.md +++ b/documentation/configuration.md @@ -22,6 +22,22 @@ await configure({ }); ``` +If your application runs entirely synchronously (for example, during early Node.js bootstrap), you can use `configureSync` instead. It follows the same shape as `configure`, but it throws if any backend requires asynchronous initialization. + +```ts +import { configureSync, InMemory } from '@zenfs/core'; + +configureSync({ + mounts: { + '/tmp': { backend: InMemory, label: 'temp-storage' }, + }, +}); +``` + +Backends that do all their work eagerly, such as `InMemory` and `SingleBuffer`, are designed to work with these synchronous configuration helpers. + +For single-mount scenarios there are matching helpers: use `configureSingle` (or `configureSingleSync`) to replace the root mount without providing a full configuration object. + ## Mounting File Systems and `resolveMountConfig` Mounting file systems in ZenFS is handled dynamically. When a mount configuration is provided, it is processed using `resolveMountConfig`, which determines how the backend should be initialized and mounted. @@ -39,6 +55,8 @@ const tmpfs = await resolveMountConfig({ mount('/mnt/tmp', tmpfs); ``` +When dealing exclusively with synchronous backends, you can call `resolveMountConfigSync`. It performs the same validation, but it throws if a backend performs any asynchronous work during creation or readiness. + ### Dynamic Mounting Mounts can be resolved dynamically at runtime, allowing flexibility when configuring storage. This is especially useful for: diff --git a/src/backends/cow.ts b/src/backends/cow.ts index 18285d73..795aaec0 100644 --- a/src/backends/cow.ts +++ b/src/backends/cow.ts @@ -8,7 +8,7 @@ import { withErrno } from 'kerium'; import { debug, err, warn } from 'kerium/log'; import { canary } from 'utilium'; import { resolveMountConfig, type MountConfiguration } from '../config.js'; -import { FileSystem } from '../internal/filesystem.js'; +import { FileSystem, ensureReadySync } from '../internal/filesystem.js'; import { isDirectory } from '../internal/inode.js'; import { dirname, join } from '../path.js'; @@ -129,6 +129,11 @@ export class CopyOnWriteFS extends FileSystem { await this.writable.ready(); } + public readySync(): void { + ensureReadySync(this.readable); + ensureReadySync(this.writable); + } + public constructor( /** The file system that initially populates this file system. */ public readonly readable: FileSystem, diff --git a/src/backends/store/fs.ts b/src/backends/store/fs.ts index 6ede1ce2..ff98c9ed 100644 --- a/src/backends/store/fs.ts +++ b/src/backends/store/fs.ts @@ -100,6 +100,18 @@ export class StoreFS extends FileSystem { this._initialized = true; } + public readySync(): void { + if (this._initialized) return; + + if (!this.attributes.has('no_async_preload')) { + this.checkRootSync(); + } + + this.checkRootSync(); + this._populateSync(); + this._initialized = true; + } + public constructor(protected readonly store: T) { super(store.type ?? 0x6b766673, store.name); store.fs = this; @@ -617,6 +629,74 @@ export class StoreFS extends FileSystem { debug(`Added ${i} existing inode(s) from store`); } + private _populateSync(): void { + if (this._initialized) { + warn('Attempted to populate tables after initialization'); + return; + } + debug('Populating tables with existing store metadata'); + using tx = this.transaction(); + + const rootData = tx.getSync(rootIno); + if (!rootData) { + notice('Store does not have a root inode'); + const inode = new Inode({ ino: rootIno, data: 1, mode: 0o777 | S_IFDIR }); + tx.setSync(inode.data, encodeUTF8('{}')); + this._add(rootIno, '/'); + tx.setSync(rootIno, inode); + tx.commitSync(); + return; + } + + if (rootData.length < sizeof(Inode)) { + crit('Store contains an invalid root inode. Refusing to populate tables'); + return; + } + + const visitedDirectories = new Set(); + let i = 0; + const queue: Array<[path: string, ino: number]> = [['/', rootIno]]; + + while (queue.length > 0) { + i++; + const [path, ino] = queue.shift()!; + + this._add(ino, path); + + const inodeData = tx.getSync(ino); + if (!inodeData) { + warn('Store is missing data for inode: ' + ino); + continue; + } + + if (inodeData.length < sizeof(Inode)) { + warn(`Invalid inode size for ino ${ino}: ${inodeData.length}`); + continue; + } + + const inode = new Inode(inodeData); + + if ((inode.mode & S_IFDIR) != S_IFDIR || visitedDirectories.has(ino)) { + continue; + } + + visitedDirectories.add(ino); + + const dirData = tx.getSync(inode.data); + if (!dirData) { + warn('Store is missing directory data: ' + inode.data); + continue; + } + const dirListing = decodeDirListing(dirData); + + for (const [entryName, childIno] of Object.entries(dirListing)) { + queue.push([join(path, entryName), childIno]); + } + } + + debug(`Added ${i} existing inode(s) from store`); + } + /** * Find an inode without using the ID tables */ diff --git a/src/config.ts b/src/config.ts index e80c93b8..aa128e9a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,8 +7,9 @@ import { checkOptions, isBackend, isBackendConfig } from './backends/backend.js' import { defaultContext } from './internal/contexts.js'; import { createCredentials } from './internal/credentials.js'; import { DeviceFS } from './internal/devices.js'; -import { FileSystem } from './internal/filesystem.js'; +import { FileSystem, ensureReadySync } from './internal/filesystem.js'; import { exists, mkdir, stat } from './node/promises.js'; +import { existsSync, mkdirSync, statSync } from './node/sync.js'; import { _setAccessChecks } from './vfs/config.js'; import { mount, mounts, umount } from './vfs/shared.js'; @@ -31,6 +32,10 @@ function isMountConfig(arg: unknown): arg is MountConfigurati return isBackendConfig(arg) || isBackend(arg) || arg instanceof FileSystem; } +function isThenable(value: unknown): value is PromiseLike { + return typeof (value as PromiseLike)?.then == 'function'; +} + /** * Retrieve a file system with `configuration`. * @category Backends and Configuration @@ -80,6 +85,60 @@ export async function resolveMountConfig(configuration: Mount return mount; } +export function resolveMountConfigSync(configuration: MountConfiguration, _depth = 0): FilesystemOf { + if (typeof configuration !== 'object' || configuration == null) { + throw log.err(withErrno('EINVAL', 'Invalid options on mount configuration')); + } + + if (!isMountConfig(configuration)) { + throw log.err(withErrno('EINVAL', 'Invalid mount configuration')); + } + + if (configuration instanceof FileSystem) { + ensureReadySync(configuration); + return configuration as FilesystemOf; + } + + if (isBackend(configuration)) { + configuration = { backend: configuration } as BackendConfiguration; + } + + for (const [key, value] of Object.entries(configuration)) { + if (key == 'backend') continue; + if (!isMountConfig(value)) continue; + + log.info('Resolving nested mount configuration: ' + key); + + if (_depth > 10) { + throw log.err(withErrno('EINVAL', 'Invalid configuration, too deep and possibly infinite')); + } + + (configuration as Record)[key] = resolveMountConfigSync(value, ++_depth); + } + + const { backend } = configuration; + + if (typeof backend.isAvailable == 'function') { + const available = backend.isAvailable(configuration as any); + if (isThenable(available)) { + throw log.err(withErrno('ENOTSUP', 'Backend availability check is asynchronous: ' + backend.name)); + } + if (!available) { + throw log.err(withErrno('EPERM', 'Backend not available: ' + backend.name)); + } + } + + checkOptions(backend, configuration); + const mountFs = backend.create(configuration); + if (isThenable(mountFs)) { + throw log.err(withErrno('ENOTSUP', 'Backend requires asynchronous initialization: ' + backend.name)); + } + const resolved = mountFs as FilesystemOf; + configureFileSystem(resolved, configuration); + ensureReadySync(resolved); + return resolved; +} + /** * An object mapping mount points to backends * @category Backends and Configuration @@ -160,6 +219,16 @@ export async function configureSingle(configuration: MountCon mount('/', resolved); } +export function configureSingleSync(configuration: MountConfiguration): void { + if (!isMountConfig(configuration)) { + throw new TypeError('Invalid single mount point configuration'); + } + + const resolved = resolveMountConfigSync(configuration); + umount('/'); + mount('/', resolved); +} + /** * Like `fs.mount`, but it also creates missing directories. * @privateRemarks @@ -181,6 +250,27 @@ async function mountWithMkdir(path: string, fs: FileSystem): Promise { mount(path, fs); } +function mountWithMkdirSync(path: string, fs: FileSystem): void { + if (path == '/') { + mount(path, fs); + return; + } + + let stats: { isDirectory(): boolean } | null = null; + try { + stats = statSync(path); + } catch (error: any) { + if (error?.code != 'ENOENT') throw error; + } + + if (!stats) { + mkdirSync(path, { recursive: true }); + } else if (!stats.isDirectory()) { + throw withErrno('ENOTDIR', 'Missing directory at mount point: ' + path); + } + mount(path, fs); +} + /** * @category Backends and Configuration */ @@ -247,6 +337,57 @@ export async function configure(configuration: Partial(configuration: Partial>): void { + Object.assign( + defaultContext.credentials, + createCredentials({ + uid: configuration.uid || 0, + gid: configuration.gid || 0, + }) + ); + + _setAccessChecks(!configuration.disableAccessChecks); + + if (configuration.log) log.configure(configuration.log); + + if (configuration.mounts) { + for (const [_point, mountConfig] of Object.entries(configuration.mounts).sort(([a], [b]) => (a.length > b.length ? 1 : -1))) { + const point = _point.startsWith('/') ? _point : '/' + _point; + + if (isBackendConfig(mountConfig)) { + mountConfig.disableAsyncCache ??= configuration.disableAsyncCache || false; + mountConfig.caseFold ??= configuration.caseFold; + } + + if (point == '/') umount('/'); + + mountWithMkdirSync(point, resolveMountConfigSync(mountConfig)); + } + } + + for (const fs of mounts.values()) { + configureFileSystem(fs, configuration); + } + + if (configuration.addDevices && !mounts.has('/dev')) { + const devfs = new DeviceFS(); + devfs.addDefaults(); + ensureReadySync(devfs); + mountWithMkdirSync('/dev', devfs); + } + + if (configuration.defaultDirectories) { + for (const dir of _defaultDirectories) { + if (existsSync(dir)) { + const stats = statSync(dir); + if (!stats.isDirectory()) log.warn('Default directory exists but is not a directory: ' + dir); + continue; + } + mkdirSync(dir); + } + } +} + export async function sync(): Promise { for (const fs of mounts.values()) await fs.sync(); } diff --git a/src/internal/filesystem.ts b/src/internal/filesystem.ts index 152624c1..536bc649 100644 --- a/src/internal/filesystem.ts +++ b/src/internal/filesystem.ts @@ -3,6 +3,8 @@ import type { UUID } from 'node:crypto'; import type { ConstMap } from 'utilium'; import type { InodeLike } from './inode.js'; +import { withErrno } from 'kerium'; + /** * Usage information about a file system * @category Internals @@ -349,3 +351,15 @@ export abstract class FileSystem { }); } } + +export function ensureReadySync(fs: FileSystem): void { + const readySync = (fs as FileSystem & { readySync?: () => void }).readySync; + if (typeof readySync == 'function') { + readySync.call(fs); + return; + } + + if (fs.ready === FileSystem.prototype.ready) return; + + throw withErrno('ENOTSUP', 'Synchronous initialization is not supported by ' + fs.name); +} diff --git a/src/mixins/mutexed.ts b/src/mixins/mutexed.ts index 558761ec..d8c5b629 100644 --- a/src/mixins/mutexed.ts +++ b/src/mixins/mutexed.ts @@ -2,6 +2,7 @@ import type { UUID } from 'node:crypto'; import type { Concrete } from 'utilium'; import type { CreationOptions, FileSystem, StreamOptions, UsageInfo } from '../internal/filesystem.js'; +import { ensureReadySync } from '../internal/filesystem.js'; import type { InodeLike } from '../internal/inode.js'; import { withErrno } from 'kerium'; @@ -83,6 +84,10 @@ export class _MutexedFS implements FileSystem { return await this._fs.ready(); } + public readySync(): void { + ensureReadySync(this._fs); + } + public usage(): UsageInfo { return this._fs.usage(); } diff --git a/tests/common/config.test.ts b/tests/common/config.test.ts new file mode 100644 index 00000000..601794bb --- /dev/null +++ b/tests/common/config.test.ts @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +import { configure, configureSingle, configureSingleSync, configureSync, fs, InMemory, mounts, SingleBuffer, type Backend } from '@zenfs/core'; +import assert from 'node:assert/strict'; +import { suite, test } from 'node:test'; + +const AsyncBackend = { + name: 'AsyncBackend', + options: {}, + async create() { + await Promise.resolve(); + return InMemory.create({ label: 'async-backend' }); + }, +} satisfies Backend; + +suite('Sync configuration', () => { + test('configureSingleSync mounts root synchronously', async () => { + configureSingleSync({ backend: InMemory, label: 'sync-root' }); + assert.equal(mounts.get('/')?.label, 'sync-root'); + + fs.writeFileSync('/sync-file', 'sync'); + assert.equal(fs.readFileSync('/sync-file', 'utf8'), 'sync'); + + await configureSingle({ backend: InMemory }); + }); + + test('configureSync mounts additional directories', async () => { + configureSync({ + mounts: { + tmp: { backend: InMemory, label: 'sync-tmp' }, + }, + defaultDirectories: true, + }); + + assert.ok(mounts.has('/tmp')); + fs.writeFileSync('/tmp/sync.txt', 'ok'); + assert.equal(fs.readFileSync('/tmp/sync.txt', 'utf8'), 'ok'); + + fs.umount('/tmp'); + fs.rmSync('/tmp', { recursive: true, force: true }); + await configureSingle({ backend: InMemory }); + }); + + test('configureSync rejects asynchronous backends', async () => { + await configure({ mounts: { '/': InMemory } }); + assert.throws(() => { + configureSync({ + mounts: { + '/': { backend: AsyncBackend }, + }, + }); + }, /asynchronous initialization/i); + }); + + test('configureSingleSync works with SingleBuffer', async () => { + const buffer = new ArrayBuffer(0x20000); + configureSingleSync({ + backend: SingleBuffer, + buffer, + }); + + fs.writeFileSync('/sb.txt', 'single-buffer'); + assert.equal(fs.readFileSync('/sb.txt', 'utf8'), 'single-buffer'); + + await configureSingle({ backend: InMemory }); + }); +}); From 78c5a50047e4967677a71e2c88ecbf753b92f991 Mon Sep 17 00:00:00 2001 From: Marcelo Lv Cabral Date: Fri, 19 Dec 2025 12:00:28 -0700 Subject: [PATCH 2/2] Prevent usage of `any` --- src/backends/cow.ts | 6 +++--- src/config.ts | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/backends/cow.ts b/src/backends/cow.ts index 795aaec0..43d6ce95 100644 --- a/src/backends/cow.ts +++ b/src/backends/cow.ts @@ -18,10 +18,10 @@ import { dirname, join } from '../path.js'; */ export interface CopyOnWriteOptions { /** The file system that initially populates this file system. */ - readable: MountConfiguration; + readable: MountConfiguration; /** The file system to write modified files to. */ - writable: MountConfiguration; + writable: MountConfiguration; /** @see {@link Journal} */ journal?: Journal; @@ -37,7 +37,7 @@ export type JournalOperation = (typeof journalOperations)[number]; /** Because TS doesn't work right w/o it */ function isJournalOp(op: string): op is JournalOperation { - return journalOperations.includes(op as any); + return journalOperations.some(operation => operation === op); } const maxOpLength = Math.max(...journalOperations.map(op => op.length)); diff --git a/src/config.ts b/src/config.ts index aa128e9a..e1766260 100644 --- a/src/config.ts +++ b/src/config.ts @@ -116,10 +116,11 @@ export function resolveMountConfigSync(configuration: MountCo (configuration as Record)[key] = resolveMountConfigSync(value, ++_depth); } - const { backend } = configuration; + const backendConfig = configuration as BackendConfiguration; + const { backend } = backendConfig; if (typeof backend.isAvailable == 'function') { - const available = backend.isAvailable(configuration as any); + const available = backend.isAvailable(backendConfig); if (isThenable(available)) { throw log.err(withErrno('ENOTSUP', 'Backend availability check is asynchronous: ' + backend.name)); } @@ -128,13 +129,13 @@ export function resolveMountConfigSync(configuration: MountCo } } - checkOptions(backend, configuration); - const mountFs = backend.create(configuration); + checkOptions(backend, backendConfig); + const mountFs = backend.create(backendConfig); if (isThenable(mountFs)) { throw log.err(withErrno('ENOTSUP', 'Backend requires asynchronous initialization: ' + backend.name)); } const resolved = mountFs as FilesystemOf; - configureFileSystem(resolved, configuration); + configureFileSystem(resolved, backendConfig); ensureReadySync(resolved); return resolved; }