From 66cc2a7fe71d11ad0ef1efcdd2459a8d3b883c84 Mon Sep 17 00:00:00 2001 From: Aron Jones Date: Wed, 28 Jan 2026 17:02:18 -0800 Subject: [PATCH 1/4] Add LazyFs - lazy-loading filesystem implementation Implements a new filesystem that loads content on-demand via user-provided loader functions. Content is cached in memory using InMemoryFs as the backing store. Designed for AI agents needing lazy access to remote or virtual content. Features: - Lazy loading of files and directories via listDir/loadFile callbacks - Caching with separate negative cache for files vs directories - Write/delete operations shadow lazy content - Full symlink support with proper resolution - Implements complete IFileSystem interface Co-Authored-By: Claude Opus 4.5 --- src/fs/lazy-fs/index.ts | 9 + src/fs/lazy-fs/lazy-fs.test.ts | 718 +++++++++++++++++++++++++++ src/fs/lazy-fs/lazy-fs.ts | 869 +++++++++++++++++++++++++++++++++ src/index.ts | 9 + 4 files changed, 1605 insertions(+) create mode 100644 src/fs/lazy-fs/index.ts create mode 100644 src/fs/lazy-fs/lazy-fs.test.ts create mode 100644 src/fs/lazy-fs/lazy-fs.ts diff --git a/src/fs/lazy-fs/index.ts b/src/fs/lazy-fs/index.ts new file mode 100644 index 00000000..861a71a9 --- /dev/null +++ b/src/fs/lazy-fs/index.ts @@ -0,0 +1,9 @@ +export { + type LazyDirEntry, + type LazyDirListing, + type LazyFileContent, + LazyFs, + type LazyFsOptions, + type LazyListDir, + type LazyLoadFile, +} from "./lazy-fs.js"; diff --git a/src/fs/lazy-fs/lazy-fs.test.ts b/src/fs/lazy-fs/lazy-fs.test.ts new file mode 100644 index 00000000..2ad36e7d --- /dev/null +++ b/src/fs/lazy-fs/lazy-fs.test.ts @@ -0,0 +1,718 @@ +import { describe, expect, it, vi } from "vitest"; +import type { LazyDirEntry, LazyFileContent } from "./lazy-fs.js"; +import { LazyFs } from "./lazy-fs.js"; + +describe("LazyFs", () => { + describe("file loading", () => { + it("should call loadFile on first file access", async () => { + const loadFile = vi.fn().mockResolvedValue({ + content: "hello world", + }); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + const content = await fs.readFile("/test.txt"); + + expect(content).toBe("hello world"); + expect(loadFile).toHaveBeenCalledWith("/test.txt"); + expect(loadFile).toHaveBeenCalledTimes(1); + }); + + it("should cache file content after first load", async () => { + const loadFile = vi.fn().mockResolvedValue({ + content: "cached content", + }); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.readFile("/test.txt"); + await fs.readFile("/test.txt"); + await fs.readFile("/test.txt"); + + expect(loadFile).toHaveBeenCalledTimes(1); + }); + + it("should return ENOENT when loadFile returns null", async () => { + const loadFile = vi.fn().mockResolvedValue(null); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + await expect(fs.readFile("/missing.txt")).rejects.toThrow("ENOENT"); + }); + + it("should cache negative results (file not found)", async () => { + const loadFile = vi.fn().mockResolvedValue(null); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + await expect(fs.readFile("/missing.txt")).rejects.toThrow("ENOENT"); + await expect(fs.readFile("/missing.txt")).rejects.toThrow("ENOENT"); + + expect(loadFile).toHaveBeenCalledTimes(1); + }); + + it("should handle binary content", async () => { + const binaryData = new Uint8Array([0x00, 0x01, 0x02, 0xff]); + const loadFile = vi.fn().mockResolvedValue({ + content: binaryData, + }); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + const result = await fs.readFileBuffer("/binary.bin"); + + expect(result).toEqual(binaryData); + }); + + it("should respect mode from loadFile result", async () => { + const loadFile = vi.fn().mockResolvedValue({ + content: "executable", + mode: 0o755, + }); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + const stat = await fs.stat("/script.sh"); + + expect(stat.mode).toBe(0o755); + }); + + it("should respect mtime from loadFile result", async () => { + const mtime = new Date("2024-01-15T10:30:00Z"); + const loadFile = vi.fn().mockResolvedValue({ + content: "content", + mtime, + }); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + const stat = await fs.stat("/file.txt"); + + expect(stat.mtime.getTime()).toBe(mtime.getTime()); + }); + }); + + describe("directory loading", () => { + it("should call listDir on readdir", async () => { + const listDir = vi.fn().mockResolvedValue([ + { name: "file1.txt", type: "file" }, + { name: "file2.txt", type: "file" }, + { name: "subdir", type: "directory" }, + ] satisfies LazyDirEntry[]); + const loadFile = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + const entries = await fs.readdir("/"); + + expect(entries).toEqual(["file1.txt", "file2.txt", "subdir"]); + expect(listDir).toHaveBeenCalledWith("/"); + }); + + it("should call listDir for each readdir (gets fresh entries)", async () => { + const listDir = vi + .fn() + .mockResolvedValue([ + { name: "file.txt", type: "file" }, + ] satisfies LazyDirEntry[]); + const loadFile = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.readdir("/"); + await fs.readdir("/"); + await fs.readdir("/"); + + // listDir is called once to verify existence, then once to get entries per readdir + // After first call, dir is marked as loaded so subsequent calls only get entries + expect(listDir).toHaveBeenCalledTimes(4); // 1 for ensureDirLoaded + 3 for getDirEntries + }); + + it("should return ENOENT for non-existent directory", async () => { + const listDir = vi.fn().mockResolvedValue(null); + const loadFile = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + await expect(fs.readdir("/missing")).rejects.toThrow("ENOENT"); + }); + + it("should return entries with correct types via readdirWithFileTypes", async () => { + const listDir = vi.fn().mockResolvedValue([ + { name: "file.txt", type: "file" }, + { name: "dir", type: "directory" }, + { name: "link", type: "symlink" }, + ] satisfies LazyDirEntry[]); + const loadFile = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + const entries = await fs.readdirWithFileTypes("/"); + + expect(entries).toHaveLength(3); + + const file = entries.find((e) => e.name === "file.txt"); + expect(file?.isFile).toBe(true); + expect(file?.isDirectory).toBe(false); + expect(file?.isSymbolicLink).toBe(false); + + const dir = entries.find((e) => e.name === "dir"); + expect(dir?.isFile).toBe(false); + expect(dir?.isDirectory).toBe(true); + expect(dir?.isSymbolicLink).toBe(false); + + const link = entries.find((e) => e.name === "link"); + expect(link?.isFile).toBe(false); + expect(link?.isDirectory).toBe(false); + expect(link?.isSymbolicLink).toBe(true); + }); + + it("should sort entries", async () => { + const listDir = vi.fn().mockResolvedValue([ + { name: "zebra.txt", type: "file" }, + { name: "apple.txt", type: "file" }, + { name: "Banana.txt", type: "file" }, + ] satisfies LazyDirEntry[]); + const loadFile = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + const entries = await fs.readdir("/"); + + // Case-sensitive sort + expect(entries).toEqual(["Banana.txt", "apple.txt", "zebra.txt"]); + }); + }); + + describe("write operations", () => { + it("should write files without calling loader", async () => { + const loadFile = vi.fn().mockResolvedValue(null); + const listDir = vi.fn().mockResolvedValue([]); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.writeFile("/new.txt", "new content"); + const content = await fs.readFile("/new.txt"); + + expect(content).toBe("new content"); + expect(loadFile).not.toHaveBeenCalledWith("/new.txt"); + }); + + it("should shadow lazy content after write", async () => { + const loadFile = vi.fn().mockResolvedValue({ + content: "original", + }); + const listDir = vi + .fn() + .mockResolvedValue([{ name: "file.txt", type: "file" }]); + + const fs = new LazyFs({ loadFile, listDir }); + + // First read loads from loader + const original = await fs.readFile("/file.txt"); + expect(original).toBe("original"); + + // Write shadows the lazy content + await fs.writeFile("/file.txt", "modified"); + + // Subsequent reads return modified content + const modified = await fs.readFile("/file.txt"); + expect(modified).toBe("modified"); + + // Loader should only be called once (before write) + expect(loadFile).toHaveBeenCalledTimes(1); + }); + + it("should append to lazy-loaded files", async () => { + const loadFile = vi.fn().mockResolvedValue({ + content: "hello", + }); + const listDir = vi.fn().mockResolvedValue([]); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.appendFile("/file.txt", " world"); + const content = await fs.readFile("/file.txt"); + + expect(content).toBe("hello world"); + }); + + it("should throw EROFS when writes disabled", async () => { + const loadFile = vi.fn().mockResolvedValue(null); + const listDir = vi.fn().mockResolvedValue([]); + + const fs = new LazyFs({ loadFile, listDir, allowWrites: false }); + + await expect(fs.writeFile("/file.txt", "content")).rejects.toThrow( + "EROFS", + ); + }); + }); + + describe("delete operations", () => { + it("should shadow lazy content after delete", async () => { + const loadFile = vi.fn().mockResolvedValue({ + content: "content", + }); + const listDir = vi + .fn() + .mockResolvedValue([{ name: "file.txt", type: "file" }]); + + const fs = new LazyFs({ loadFile, listDir }); + + // Verify file exists initially + expect(await fs.exists("/file.txt")).toBe(true); + + // Delete + await fs.rm("/file.txt"); + + // File should no longer exist + expect(await fs.exists("/file.txt")).toBe(false); + await expect(fs.readFile("/file.txt")).rejects.toThrow("ENOENT"); + }); + + it("should not show deleted files in readdir", async () => { + const listDir = vi.fn().mockResolvedValue([ + { name: "file1.txt", type: "file" }, + { name: "file2.txt", type: "file" }, + ]); + const loadFile = vi.fn().mockResolvedValue({ content: "x" }); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.rm("/file1.txt"); + + const entries = await fs.readdir("/"); + expect(entries).toEqual(["file2.txt"]); + }); + + it("should allow recreating deleted files", async () => { + const loadFile = vi.fn().mockResolvedValue({ + content: "original", + }); + const listDir = vi.fn().mockResolvedValue([]); + + const fs = new LazyFs({ loadFile, listDir }); + + // Load, delete, then recreate + await fs.readFile("/file.txt"); + await fs.rm("/file.txt"); + await fs.writeFile("/file.txt", "new content"); + + const content = await fs.readFile("/file.txt"); + expect(content).toBe("new content"); + }); + }); + + describe("stat operations", () => { + it("should identify files vs directories from listDir", async () => { + const listDir = vi.fn().mockImplementation(async (path: string) => { + if (path === "/") { + return [ + { name: "file.txt", type: "file" }, + { name: "dir", type: "directory" }, + ] satisfies LazyDirEntry[]; + } + if (path === "/dir") { + return []; + } + return null; + }); + const loadFile = vi.fn().mockImplementation(async (path: string) => { + if (path === "/file.txt") { + return { content: "content" }; + } + return null; + }); + + const fs = new LazyFs({ loadFile, listDir }); + + const fileStat = await fs.stat("/file.txt"); + expect(fileStat.isFile).toBe(true); + expect(fileStat.isDirectory).toBe(false); + + const dirStat = await fs.stat("/dir"); + expect(dirStat.isFile).toBe(false); + expect(dirStat.isDirectory).toBe(true); + }); + + it("should return correct size for loaded files", async () => { + const loadFile = vi.fn().mockResolvedValue({ + content: "hello", // 5 bytes + }); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + const stat = await fs.stat("/file.txt"); + + expect(stat.size).toBe(5); + }); + }); + + describe("symlink handling", () => { + it("should load symlinks correctly", async () => { + const loadFile = vi.fn().mockImplementation(async (path: string) => { + if (path === "/link") { + return { + content: "/target.txt", + isSymlink: true, + } satisfies LazyFileContent; + } + if (path === "/target.txt") { + return { content: "target content" }; + } + return null; + }); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + // Reading the symlink should follow to target + const content = await fs.readFile("/link"); + expect(content).toBe("target content"); + }); + + it("should identify symlinks with lstat", async () => { + const loadFile = vi.fn().mockResolvedValue({ + content: "/target", + isSymlink: true, + }); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + const stat = await fs.lstat("/link"); + + expect(stat.isSymbolicLink).toBe(true); + expect(stat.isFile).toBe(false); + }); + + it("should create symlinks locally", async () => { + const loadFile = vi.fn().mockResolvedValue(null); + const listDir = vi.fn().mockImplementation(async (path: string) => { + // Only root exists as a directory + if (path === "/") { + return []; + } + return null; + }); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.writeFile("/target.txt", "content"); + await fs.symlink("/target.txt", "/link"); + + const target = await fs.readlink("/link"); + expect(target).toBe("/target.txt"); + + const content = await fs.readFile("/link"); + expect(content).toBe("content"); + }); + }); + + describe("mkdir", () => { + it("should create directories locally", async () => { + const loadFile = vi.fn().mockResolvedValue(null); + const listDir = vi.fn().mockImplementation(async (path: string) => { + // Only root exists as a directory + if (path === "/") { + return []; + } + return null; + }); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.mkdir("/newdir"); + + const stat = await fs.stat("/newdir"); + expect(stat.isDirectory).toBe(true); + }); + + it("should create nested directories with recursive option", async () => { + const loadFile = vi.fn().mockResolvedValue(null); + const listDir = vi.fn().mockImplementation(async (path: string) => { + // Only root exists as a directory + if (path === "/") { + return []; + } + return null; + }); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.mkdir("/a/b/c", { recursive: true }); + + expect(await fs.exists("/a")).toBe(true); + expect(await fs.exists("/a/b")).toBe(true); + expect(await fs.exists("/a/b/c")).toBe(true); + }); + + it("should throw EEXIST for existing directory without recursive", async () => { + const loadFile = vi.fn().mockResolvedValue(null); + const listDir = vi.fn().mockImplementation(async (path: string) => { + // Only root exists as a directory initially + if (path === "/") { + return []; + } + return null; + }); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.mkdir("/dir"); + + await expect(fs.mkdir("/dir")).rejects.toThrow("EEXIST"); + }); + }); + + describe("exists", () => { + it("should return true for loaded files", async () => { + const loadFile = vi.fn().mockResolvedValue({ content: "x" }); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + expect(await fs.exists("/file.txt")).toBe(true); + }); + + it("should return true for loaded directories", async () => { + const loadFile = vi.fn().mockResolvedValue(null); + const listDir = vi.fn().mockResolvedValue([]); + + const fs = new LazyFs({ loadFile, listDir }); + + expect(await fs.exists("/dir")).toBe(true); + }); + + it("should return false for non-existent paths", async () => { + const loadFile = vi.fn().mockResolvedValue(null); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + expect(await fs.exists("/missing")).toBe(false); + }); + + it("should return false for deleted paths", async () => { + const loadFile = vi.fn().mockResolvedValue({ content: "x" }); + const listDir = vi.fn().mockResolvedValue([]); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.rm("/file.txt"); + + expect(await fs.exists("/file.txt")).toBe(false); + }); + }); + + describe("cp and mv", () => { + it("should copy lazy-loaded files", async () => { + const loadFile = vi.fn().mockResolvedValue({ content: "copied" }); + const listDir = vi.fn().mockResolvedValue([]); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.cp("/src.txt", "/dst.txt"); + + const content = await fs.readFile("/dst.txt"); + expect(content).toBe("copied"); + }); + + it("should move lazy-loaded files", async () => { + const loadFile = vi.fn().mockResolvedValue({ content: "moved" }); + const listDir = vi.fn().mockResolvedValue([]); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.mv("/src.txt", "/dst.txt"); + + expect(await fs.exists("/src.txt")).toBe(false); + const content = await fs.readFile("/dst.txt"); + expect(content).toBe("moved"); + }); + }); + + describe("chmod and utimes", () => { + it("should change mode of lazy-loaded file", async () => { + const loadFile = vi.fn().mockResolvedValue({ + content: "x", + mode: 0o644, + }); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.chmod("/file.txt", 0o755); + + const stat = await fs.stat("/file.txt"); + expect(stat.mode).toBe(0o755); + }); + + it("should change mtime of lazy-loaded file", async () => { + const loadFile = vi.fn().mockResolvedValue({ content: "x" }); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + const newTime = new Date("2025-01-01T00:00:00Z"); + await fs.utimes("/file.txt", newTime, newTime); + + const stat = await fs.stat("/file.txt"); + expect(stat.mtime.getTime()).toBe(newTime.getTime()); + }); + }); + + describe("link and readlink", () => { + it("should create hard links", async () => { + const loadFile = vi.fn().mockImplementation(async (path: string) => { + if (path === "/src.txt") { + return { content: "content" }; + } + return null; + }); + const listDir = vi.fn().mockImplementation(async (path: string) => { + // Only root exists as a directory + if (path === "/") { + return []; + } + return null; + }); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.link("/src.txt", "/hardlink.txt"); + + const content = await fs.readFile("/hardlink.txt"); + expect(content).toBe("content"); + }); + + it("should read symlink targets", async () => { + const loadFile = vi.fn().mockResolvedValue({ + content: "/target", + isSymlink: true, + }); + const listDir = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + const target = await fs.readlink("/link"); + expect(target).toBe("/target"); + }); + }); + + describe("realpath", () => { + it("should resolve symlinks in path", async () => { + const loadFile = vi.fn().mockImplementation(async (path: string) => { + if (path === "/link") { + return { content: "/real", isSymlink: true }; + } + return null; + }); + const listDir = vi.fn().mockImplementation(async (path: string) => { + if (path === "/real") { + return []; + } + return null; + }); + + const fs = new LazyFs({ loadFile, listDir }); + + const resolved = await fs.realpath("/link"); + expect(resolved).toBe("/real"); + }); + }); + + describe("resolvePath", () => { + it("should resolve relative paths", () => { + const fs = new LazyFs({ + loadFile: async () => null, + listDir: async () => null, + }); + + expect(fs.resolvePath("/home", "file.txt")).toBe("/home/file.txt"); + expect(fs.resolvePath("/home/user", "../file.txt")).toBe( + "/home/file.txt", + ); + expect(fs.resolvePath("/", "file.txt")).toBe("/file.txt"); + }); + + it("should handle absolute paths", () => { + const fs = new LazyFs({ + loadFile: async () => null, + listDir: async () => null, + }); + + expect(fs.resolvePath("/home", "/etc/passwd")).toBe("/etc/passwd"); + }); + }); + + describe("getAllPaths", () => { + it("should return loaded and modified paths", async () => { + const loadFile = vi.fn().mockImplementation(async (path: string) => { + if (path === "/loaded.txt") { + return { content: "x" }; + } + return null; + }); + const listDir = vi.fn().mockImplementation(async (path: string) => { + if (path === "/") { + return []; + } + return null; + }); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.readFile("/loaded.txt"); + await fs.writeFile("/written.txt", "y"); + await fs.mkdir("/dir"); + + const paths = fs.getAllPaths(); + + expect(paths).toContain("/loaded.txt"); + expect(paths).toContain("/written.txt"); + expect(paths).toContain("/dir"); + }); + + it("should not include deleted paths", async () => { + const loadFile = vi.fn().mockResolvedValue({ content: "x" }); + const listDir = vi.fn().mockResolvedValue([]); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.readFile("/file.txt"); + await fs.rm("/file.txt"); + + const paths = fs.getAllPaths(); + + expect(paths).not.toContain("/file.txt"); + }); + }); + + describe("locally added files in readdir", () => { + it("should include locally written files in directory listing", async () => { + const listDir = vi + .fn() + .mockResolvedValue([{ name: "remote.txt", type: "file" }]); + const loadFile = vi.fn().mockResolvedValue(null); + + const fs = new LazyFs({ loadFile, listDir }); + + await fs.writeFile("/local.txt", "local content"); + + const entries = await fs.readdir("/"); + + expect(entries).toContain("remote.txt"); + expect(entries).toContain("local.txt"); + }); + }); +}); diff --git a/src/fs/lazy-fs/lazy-fs.ts b/src/fs/lazy-fs/lazy-fs.ts new file mode 100644 index 00000000..54fd9165 --- /dev/null +++ b/src/fs/lazy-fs/lazy-fs.ts @@ -0,0 +1,869 @@ +/** + * LazyFs - Lazy-loading filesystem backed by user-provided loader functions + * + * Content is loaded on-demand and cached in an InMemoryFs backing store. + * Designed for AI agents needing lazy access to remote or virtual content. + */ + +import { + type FileContent, + fromBuffer, + getEncoding, + toBuffer, +} from "../encoding.js"; +import { InMemoryFs } from "../in-memory-fs/in-memory-fs.js"; +import type { + CpOptions, + DirentEntry, + FsStat, + IFileSystem, + MkdirOptions, + ReadFileOptions, + RmOptions, + WriteFileOptions, +} from "../interface.js"; + +/** + * Entry type in directory listing + */ +export interface LazyDirEntry { + name: string; + type: "file" | "directory" | "symlink"; +} + +/** + * Result from listDir - array of entries or null if dir doesn't exist + */ +export type LazyDirListing = LazyDirEntry[] | null; + +/** + * Function to list directory contents + */ +export type LazyListDir = (dirPath: string) => Promise; + +/** + * Result from loadFile + */ +export interface LazyFileContent { + content: string | Uint8Array; + mode?: number; // Default: 0o644 + mtime?: Date; + isSymlink?: boolean; // If true, content is treated as symlink target +} + +/** + * Function to load file content - returns content or null if doesn't exist + */ +export type LazyLoadFile = ( + filePath: string, +) => Promise; + +export interface LazyFsOptions { + listDir: LazyListDir; + loadFile: LazyLoadFile; + allowWrites?: boolean; // Default: true +} + +export class LazyFs implements IFileSystem { + private readonly cache: InMemoryFs; + private readonly listDir: LazyListDir; + private readonly loadFile: LazyLoadFile; + private readonly allowWrites: boolean; + + // Tracking state + private readonly loadedFiles: Set = new Set(); + private readonly loadedDirs: Set = new Set(); + private readonly notExistsAsFile: Set = new Set(); // Path is not a file + private readonly notExistsAsDir: Set = new Set(); // Path is not a directory + private readonly modified: Set = new Set(); + private readonly deleted: Set = new Set(); + + constructor(options: LazyFsOptions) { + this.cache = new InMemoryFs(); + this.listDir = options.listDir; + this.loadFile = options.loadFile; + this.allowWrites = options.allowWrites ?? true; + } + + private normalizePath(path: string): string { + if (!path || path === "/") return "/"; + + let normalized = + path.endsWith("/") && path !== "/" ? path.slice(0, -1) : path; + + if (!normalized.startsWith("/")) { + normalized = `/${normalized}`; + } + + const parts = normalized.split("/").filter((p) => p && p !== "."); + const resolved: string[] = []; + + for (const part of parts) { + if (part === "..") { + resolved.pop(); + } else { + resolved.push(part); + } + } + + return `/${resolved.join("/")}` || "/"; + } + + private dirname(path: string): string { + const normalized = this.normalizePath(path); + if (normalized === "/") return "/"; + const lastSlash = normalized.lastIndexOf("/"); + return lastSlash === 0 ? "/" : normalized.slice(0, lastSlash); + } + + private assertWritable(operation: string): void { + if (!this.allowWrites) { + throw new Error(`EROFS: read-only file system, ${operation}`); + } + } + + /** + * Ensure a file has been loaded from the lazy loader + */ + private async ensureFileLoaded(path: string): Promise { + const normalized = this.normalizePath(path); + + // If deleted locally, it doesn't exist + if (this.deleted.has(normalized)) { + return false; + } + + // If modified locally, use cached version + if (this.modified.has(normalized)) { + return true; + } + + // If already loaded or known not to exist as a file + if (this.loadedFiles.has(normalized)) { + return true; + } + if (this.notExistsAsFile.has(normalized)) { + return false; + } + + // Try to load + const result = await this.loadFile(normalized); + if (result === null) { + this.notExistsAsFile.add(normalized); + return false; + } + + // Store in cache + this.loadedFiles.add(normalized); + + if (result.isSymlink) { + // Content is symlink target + const target = + typeof result.content === "string" + ? result.content + : new TextDecoder().decode(result.content); + await this.cache.symlink(target, normalized); + } else { + // Regular file + const buffer = toBuffer(result.content); + await this.cache.writeFile(normalized, buffer); + + // Set metadata if provided + if (result.mode !== undefined) { + await this.cache.chmod(normalized, result.mode); + } + if (result.mtime !== undefined) { + await this.cache.utimes(normalized, result.mtime, result.mtime); + } + } + + return true; + } + + /** + * Ensure a directory listing has been loaded + */ + private async ensureDirLoaded(path: string): Promise { + const normalized = this.normalizePath(path); + + // If deleted locally, it doesn't exist + if (this.deleted.has(normalized)) { + return false; + } + + // If modified locally (created as dir), use cached version + if (this.modified.has(normalized)) { + return true; + } + + // If already loaded or known not to exist as a directory + if (this.loadedDirs.has(normalized)) { + return true; + } + if (this.notExistsAsDir.has(normalized)) { + return false; + } + + // Try to load + const entries = await this.listDir(normalized); + if (entries === null) { + this.notExistsAsDir.add(normalized); + return false; + } + + // Create directory in cache if needed + try { + const exists = await this.cache.exists(normalized); + if (!exists) { + await this.cache.mkdir(normalized, { recursive: true }); + } + } catch { + // Directory might already exist + } + + // Store entries as placeholders (will be loaded on demand) + for (const entry of entries) { + const childPath = + normalized === "/" ? `/${entry.name}` : `${normalized}/${entry.name}`; + + // Don't overwrite locally modified/loaded entries + if ( + this.modified.has(childPath) || + this.loadedFiles.has(childPath) || + this.loadedDirs.has(childPath) + ) { + continue; + } + + // Create placeholder based on type + if (entry.type === "directory") { + try { + const exists = await this.cache.exists(childPath); + if (!exists) { + await this.cache.mkdir(childPath, { recursive: true }); + } + } catch { + // Directory might already exist + } + } else if (entry.type === "symlink") { + // For symlinks, we need to load the actual file to get the target + // Just mark it as needing load + } else { + // For files, create an empty placeholder that will be replaced on read + // We don't create files here - they'll be loaded on demand + } + } + + this.loadedDirs.add(normalized); + return true; + } + + /** + * Get entries for a directory from the loaded listing + */ + private async getDirEntries(path: string): Promise { + const normalized = this.normalizePath(path); + + // First ensure the directory is loaded + const exists = await this.ensureDirLoaded(normalized); + if (!exists) { + return null; + } + + // Re-fetch from loader to get accurate entries + const entries = await this.listDir(normalized); + return entries; + } + + async readFile( + path: string, + options?: ReadFileOptions | BufferEncoding, + ): Promise { + const buffer = await this.readFileBuffer(path); + const encoding = getEncoding(options); + return fromBuffer(buffer, encoding); + } + + async readFileBuffer(path: string): Promise { + const normalized = this.normalizePath(path); + + // Check if deleted + if (this.deleted.has(normalized)) { + throw new Error(`ENOENT: no such file or directory, open '${path}'`); + } + + // If modified locally, read from cache directly + if (this.modified.has(normalized)) { + return this.cache.readFileBuffer(normalized); + } + + // Ensure file is loaded + const loaded = await this.ensureFileLoaded(normalized); + if (!loaded) { + throw new Error(`ENOENT: no such file or directory, open '${path}'`); + } + + // Check if it's a symlink and load the target + const stat = await this.cache.lstat(normalized); + if (stat.isSymbolicLink) { + const target = await this.cache.readlink(normalized); + const resolvedTarget = target.startsWith("/") + ? target + : this.resolvePath(this.dirname(normalized), target); + // Recursively read the target + return this.readFileBuffer(resolvedTarget); + } + + // Read from cache + return this.cache.readFileBuffer(normalized); + } + + async writeFile( + path: string, + content: FileContent, + options?: WriteFileOptions | BufferEncoding, + ): Promise { + this.assertWritable(`write '${path}'`); + const normalized = this.normalizePath(path); + + // Ensure parent directory exists + const parent = this.dirname(normalized); + if (parent !== "/") { + await this.ensureDirLoaded(parent); + } + + // Write to cache + await this.cache.writeFile(normalized, content, options); + + // Mark as modified + this.modified.add(normalized); + this.deleted.delete(normalized); + this.notExistsAsFile.delete(normalized); + this.notExistsAsDir.delete(normalized); + } + + async appendFile( + path: string, + content: FileContent, + options?: WriteFileOptions | BufferEncoding, + ): Promise { + this.assertWritable(`append '${path}'`); + const normalized = this.normalizePath(path); + + // Try to load existing content first + if (!this.modified.has(normalized)) { + await this.ensureFileLoaded(normalized); + } + + // Append to cache + await this.cache.appendFile(normalized, content, options); + + // Mark as modified + this.modified.add(normalized); + this.deleted.delete(normalized); + } + + async exists(path: string): Promise { + const normalized = this.normalizePath(path); + + // If deleted locally + if (this.deleted.has(normalized)) { + return false; + } + + // If modified locally, check cache + if (this.modified.has(normalized)) { + return this.cache.exists(normalized); + } + + // If loaded, check cache + if (this.loadedFiles.has(normalized) || this.loadedDirs.has(normalized)) { + return this.cache.exists(normalized); + } + + // If known not to exist (checked both file and dir) + if ( + this.notExistsAsFile.has(normalized) && + this.notExistsAsDir.has(normalized) + ) { + return false; + } + + // Try loading as file first + const fileLoaded = await this.ensureFileLoaded(normalized); + if (fileLoaded) { + return true; + } + + // Try loading as directory + const dirLoaded = await this.ensureDirLoaded(normalized); + return dirLoaded; + } + + async stat(path: string): Promise { + const normalized = this.normalizePath(path); + + // If deleted locally + if (this.deleted.has(normalized)) { + throw new Error(`ENOENT: no such file or directory, stat '${path}'`); + } + + // If modified locally or already loaded, use cache + if ( + this.modified.has(normalized) || + this.loadedFiles.has(normalized) || + this.loadedDirs.has(normalized) + ) { + return this.cache.stat(normalized); + } + + // Try to load file first + const fileLoaded = await this.ensureFileLoaded(normalized); + if (fileLoaded) { + return this.cache.stat(normalized); + } + + // Try to load as directory + const dirLoaded = await this.ensureDirLoaded(normalized); + if (dirLoaded) { + return this.cache.stat(normalized); + } + + throw new Error(`ENOENT: no such file or directory, stat '${path}'`); + } + + async lstat(path: string): Promise { + const normalized = this.normalizePath(path); + + // If deleted locally + if (this.deleted.has(normalized)) { + throw new Error(`ENOENT: no such file or directory, lstat '${path}'`); + } + + // If modified locally or already loaded, use cache + if ( + this.modified.has(normalized) || + this.loadedFiles.has(normalized) || + this.loadedDirs.has(normalized) + ) { + return this.cache.lstat(normalized); + } + + // Try to load file first + const fileLoaded = await this.ensureFileLoaded(normalized); + if (fileLoaded) { + return this.cache.lstat(normalized); + } + + // Try to load as directory + const dirLoaded = await this.ensureDirLoaded(normalized); + if (dirLoaded) { + return this.cache.lstat(normalized); + } + + throw new Error(`ENOENT: no such file or directory, lstat '${path}'`); + } + + async mkdir(path: string, options?: MkdirOptions): Promise { + this.assertWritable(`mkdir '${path}'`); + const normalized = this.normalizePath(path); + + // Check if already exists + const exists = await this.exists(normalized); + if (exists) { + if (!options?.recursive) { + throw new Error(`EEXIST: directory already exists, mkdir '${path}'`); + } + return; + } + + // Ensure parent exists (or create recursively) + const parent = this.dirname(normalized); + if (parent !== "/") { + if (options?.recursive) { + // Recursively create parent directories + await this.mkdir(parent, { recursive: true }); + } else { + const parentExists = await this.exists(parent); + if (!parentExists) { + throw new Error(`ENOENT: no such file or directory, mkdir '${path}'`); + } + } + } + + // Create in cache (not recursive since we handle it above) + await this.cache.mkdir(normalized); + + // Mark as modified + this.modified.add(normalized); + this.loadedDirs.add(normalized); + this.deleted.delete(normalized); + this.notExistsAsFile.delete(normalized); + this.notExistsAsDir.delete(normalized); + } + + async readdir(path: string): Promise { + const entries = await this.readdirWithFileTypes(path); + return entries.map((e) => e.name); + } + + async readdirWithFileTypes(path: string): Promise { + const normalized = this.normalizePath(path); + + // If deleted locally + if (this.deleted.has(normalized)) { + throw new Error(`ENOENT: no such file or directory, scandir '${path}'`); + } + + // Ensure directory is loaded + const loaded = await this.ensureDirLoaded(normalized); + if (!loaded) { + throw new Error(`ENOENT: no such file or directory, scandir '${path}'`); + } + + // Get entries from loader + const lazyEntries = await this.getDirEntries(normalized); + if (!lazyEntries) { + throw new Error(`ENOENT: no such file or directory, scandir '${path}'`); + } + + // Build result from lazy entries plus any locally added files + const entriesMap = new Map(); + + // Add lazy entries + for (const entry of lazyEntries) { + const childPath = + normalized === "/" ? `/${entry.name}` : `${normalized}/${entry.name}`; + + // Skip if deleted locally + if (this.deleted.has(childPath)) { + continue; + } + + entriesMap.set(entry.name, { + name: entry.name, + isFile: entry.type === "file", + isDirectory: entry.type === "directory", + isSymbolicLink: entry.type === "symlink", + }); + } + + // Add locally modified entries that are direct children + const prefix = normalized === "/" ? "/" : `${normalized}/`; + for (const modPath of this.modified) { + if (modPath.startsWith(prefix)) { + const rest = modPath.slice(prefix.length); + const name = rest.split("/")[0]; + if (name && !rest.includes("/", 1) && !entriesMap.has(name)) { + // Get type from cache + try { + const stat = await this.cache.lstat(modPath); + entriesMap.set(name, { + name, + isFile: stat.isFile, + isDirectory: stat.isDirectory, + isSymbolicLink: stat.isSymbolicLink, + }); + } catch { + // Entry doesn't exist in cache + } + } + } + } + + // Sort and return + return Array.from(entriesMap.values()).sort((a, b) => + a.name < b.name ? -1 : a.name > b.name ? 1 : 0, + ); + } + + async rm(path: string, options?: RmOptions): Promise { + this.assertWritable(`rm '${path}'`); + const normalized = this.normalizePath(path); + + // Check if exists + const exists = await this.exists(normalized); + if (!exists) { + if (options?.force) return; + throw new Error(`ENOENT: no such file or directory, rm '${path}'`); + } + + // Check if it's a directory + const stat = await this.stat(normalized); + if (stat.isDirectory) { + const children = await this.readdir(normalized); + if (children.length > 0) { + if (!options?.recursive) { + throw new Error(`ENOTEMPTY: directory not empty, rm '${path}'`); + } + for (const child of children) { + const childPath = + normalized === "/" ? `/${child}` : `${normalized}/${child}`; + await this.rm(childPath, options); + } + } + } + + // Mark as deleted + this.deleted.add(normalized); + this.modified.delete(normalized); + this.loadedFiles.delete(normalized); + this.loadedDirs.delete(normalized); + + // Remove from cache + try { + await this.cache.rm(normalized, { force: true }); + } catch { + // Ignore errors + } + } + + async cp(src: string, dest: string, options?: CpOptions): Promise { + this.assertWritable(`cp '${dest}'`); + const srcNorm = this.normalizePath(src); + const destNorm = this.normalizePath(dest); + + // Ensure source exists + const srcExists = await this.exists(srcNorm); + if (!srcExists) { + throw new Error(`ENOENT: no such file or directory, cp '${src}'`); + } + + const srcStat = await this.stat(srcNorm); + + if (srcStat.isFile) { + const content = await this.readFileBuffer(srcNorm); + await this.writeFile(destNorm, content); + } else if (srcStat.isDirectory) { + if (!options?.recursive) { + throw new Error(`EISDIR: is a directory, cp '${src}'`); + } + await this.mkdir(destNorm, { recursive: true }); + const children = await this.readdir(srcNorm); + for (const child of children) { + const srcChild = srcNorm === "/" ? `/${child}` : `${srcNorm}/${child}`; + const destChild = + destNorm === "/" ? `/${child}` : `${destNorm}/${child}`; + await this.cp(srcChild, destChild, options); + } + } + } + + async mv(src: string, dest: string): Promise { + this.assertWritable(`mv '${dest}'`); + await this.cp(src, dest, { recursive: true }); + await this.rm(src, { recursive: true }); + } + + resolvePath(base: string, path: string): string { + if (path.startsWith("/")) { + return this.normalizePath(path); + } + const combined = base === "/" ? `/${path}` : `${base}/${path}`; + return this.normalizePath(combined); + } + + getAllPaths(): string[] { + // Return all paths we know about + const paths = new Set(); + + // Add loaded files + for (const p of this.loadedFiles) { + if (!this.deleted.has(p)) { + paths.add(p); + } + } + + // Add loaded directories + for (const p of this.loadedDirs) { + if (!this.deleted.has(p)) { + paths.add(p); + } + } + + // Add modified paths + for (const p of this.modified) { + if (!this.deleted.has(p)) { + paths.add(p); + } + } + + return Array.from(paths); + } + + async chmod(path: string, mode: number): Promise { + this.assertWritable(`chmod '${path}'`); + const normalized = this.normalizePath(path); + + // Ensure file/dir is loaded + const exists = await this.exists(normalized); + if (!exists) { + throw new Error(`ENOENT: no such file or directory, chmod '${path}'`); + } + + // Load into cache if needed + if ( + !this.modified.has(normalized) && + !this.loadedFiles.has(normalized) && + !this.loadedDirs.has(normalized) + ) { + await this.ensureFileLoaded(normalized); + } + + await this.cache.chmod(normalized, mode); + this.modified.add(normalized); + } + + async symlink(target: string, linkPath: string): Promise { + this.assertWritable(`symlink '${linkPath}'`); + const normalized = this.normalizePath(linkPath); + + // Check if already exists + const exists = await this.exists(normalized); + if (exists) { + throw new Error(`EEXIST: file already exists, symlink '${linkPath}'`); + } + + // Ensure parent exists + const parent = this.dirname(normalized); + if (parent !== "/") { + await this.ensureDirLoaded(parent); + } + + await this.cache.symlink(target, normalized); + this.modified.add(normalized); + this.loadedFiles.add(normalized); + this.deleted.delete(normalized); + this.notExistsAsFile.delete(normalized); + this.notExistsAsDir.delete(normalized); + } + + async link(existingPath: string, newPath: string): Promise { + this.assertWritable(`link '${newPath}'`); + const existingNorm = this.normalizePath(existingPath); + const newNorm = this.normalizePath(newPath); + + // Ensure source exists and is a file + const srcExists = await this.exists(existingNorm); + if (!srcExists) { + throw new Error( + `ENOENT: no such file or directory, link '${existingPath}'`, + ); + } + + const srcStat = await this.stat(existingNorm); + if (!srcStat.isFile) { + throw new Error(`EPERM: operation not permitted, link '${existingPath}'`); + } + + // Check dest doesn't exist + const destExists = await this.exists(newNorm); + if (destExists) { + throw new Error(`EEXIST: file already exists, link '${newPath}'`); + } + + // Ensure parent of dest exists + const parent = this.dirname(newNorm); + if (parent !== "/") { + await this.ensureDirLoaded(parent); + } + + // Load source content and create hard link + await this.ensureFileLoaded(existingNorm); + await this.cache.link(existingNorm, newNorm); + + this.modified.add(newNorm); + this.loadedFiles.add(newNorm); + this.deleted.delete(newNorm); + this.notExistsAsFile.delete(newNorm); + this.notExistsAsDir.delete(newNorm); + } + + async readlink(path: string): Promise { + const normalized = this.normalizePath(path); + + // If deleted + if (this.deleted.has(normalized)) { + throw new Error(`ENOENT: no such file or directory, readlink '${path}'`); + } + + // Ensure file is loaded + if (!this.modified.has(normalized) && !this.loadedFiles.has(normalized)) { + const loaded = await this.ensureFileLoaded(normalized); + if (!loaded) { + throw new Error( + `ENOENT: no such file or directory, readlink '${path}'`, + ); + } + } + + return this.cache.readlink(normalized); + } + + async realpath(path: string): Promise { + const normalized = this.normalizePath(path); + + // Check if exists + const exists = await this.exists(normalized); + if (!exists) { + throw new Error(`ENOENT: no such file or directory, realpath '${path}'`); + } + + // Load into cache if needed + if ( + !this.modified.has(normalized) && + !this.loadedFiles.has(normalized) && + !this.loadedDirs.has(normalized) + ) { + await this.ensureFileLoaded(normalized); + } + + // Check if it's a symlink and resolve it + const stat = await this.cache.lstat(normalized); + if (stat.isSymbolicLink) { + const target = await this.cache.readlink(normalized); + const resolvedTarget = target.startsWith("/") + ? target + : this.resolvePath(this.dirname(normalized), target); + + // Make sure the target is loaded + const targetExists = await this.exists(resolvedTarget); + if (!targetExists) { + throw new Error( + `ENOENT: no such file or directory, realpath '${path}'`, + ); + } + + // Recursively resolve + return this.realpath(resolvedTarget); + } + + return normalized; + } + + async utimes(path: string, atime: Date, mtime: Date): Promise { + this.assertWritable(`utimes '${path}'`); + const normalized = this.normalizePath(path); + + // Ensure file/dir is loaded + const exists = await this.exists(normalized); + if (!exists) { + throw new Error(`ENOENT: no such file or directory, utimes '${path}'`); + } + + // Load into cache if needed + if ( + !this.modified.has(normalized) && + !this.loadedFiles.has(normalized) && + !this.loadedDirs.has(normalized) + ) { + await this.ensureFileLoaded(normalized); + } + + await this.cache.utimes(normalized, atime, mtime); + this.modified.add(normalized); + } +} + +// Re-export type alias +export type BufferEncoding = import("../interface.js").BufferEncoding; diff --git a/src/index.ts b/src/index.ts index 527615c3..e11c1f65 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,6 +28,15 @@ export type { RmOptions, SymlinkEntry, } from "./fs/interface.js"; +export { + type LazyDirEntry, + type LazyDirListing, + type LazyFileContent, + LazyFs, + type LazyFsOptions, + type LazyListDir, + type LazyLoadFile, +} from "./fs/lazy-fs/index.js"; export { MountableFs, type MountableFsOptions, From c95387dff019faade7117c1ce222a070ec5caddb Mon Sep 17 00:00:00 2001 From: Aron Jones Date: Wed, 28 Jan 2026 17:07:29 -0800 Subject: [PATCH 2/4] Add LazyFs documentation to README Document the new lazy-loading filesystem implementation with usage examples showing listDir/loadFile callbacks and caching behavior. Co-Authored-By: Claude Opus 4.5 --- README.md | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 99847bdf..9037b9d4 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Custom commands receive the full `CommandContext` with access to `fs`, `cwd`, `e ### Filesystem Options -Four filesystem implementations are available: +Five filesystem implementations are available: **InMemoryFs** (default) - Pure in-memory filesystem, no disk access: @@ -171,6 +171,36 @@ const fs = new MountableFs({ }); ``` +**LazyFs** - Lazy-loading filesystem that loads content on-demand via callbacks. Ideal for remote content, API-backed storage, or large datasets where you don't want to load everything upfront: + +```typescript +import { Bash, LazyFs, MountableFs, InMemoryFs } from "just-bash"; + +const lazyFs = new LazyFs({ + // Called when reading a directory + listDir: async (dirPath) => { + const entries = await fetchDirectoryFromAPI(dirPath); + return entries; // [{ name: "file.txt", type: "file" }, ...] or null + }, + // Called when reading a file + loadFile: async (filePath) => { + const content = await fetchFileFromAPI(filePath); + return content; // { content: "...", mode?, mtime? } or null + }, + allowWrites: true, // default, writes go to in-memory cache +}); + +// Mount it at a path +const fs = new MountableFs({ base: new InMemoryFs() }); +fs.mount("/mnt/remote", lazyFs); + +const bash = new Bash({ fs }); +await bash.exec("ls /mnt/remote"); // triggers listDir("/") +await bash.exec("cat /mnt/remote/data.txt"); // triggers loadFile("/data.txt") +``` + +Content is cached after first access. Writes and deletes stay in memory and shadow the lazy-loaded content. + ### AI SDK Tool For AI agents, use [`bash-tool`](https://github.com/vercel-labs/bash-tool) which is optimized for just-bash and provides a ready-to-use [AI SDK](https://ai-sdk.dev/) tool: From 215dd5fecebd962d203fc68180d5652a7e293c52 Mon Sep 17 00:00:00 2001 From: Aron Jones Date: Wed, 28 Jan 2026 17:24:01 -0800 Subject: [PATCH 3/4] Fix LazyFs README example and add .claude to gitignore - Fix TypeScript compilation error in LazyFs documentation example by using inline stub values instead of undefined placeholder functions - Use correct type "directory" instead of "dir" to match LazyDirEntry - Add .claude/ to gitignore (local Claude Code settings) Co-Authored-By: Claude Opus 4.5 --- .gitignore | 1 + README.md | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index fdc814f3..229346c0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ package-lock.json .DS_Store .vscode .idea +.claude/ .env .env.* dist diff --git a/README.md b/README.md index 9037b9d4..03ad98d9 100644 --- a/README.md +++ b/README.md @@ -177,15 +177,18 @@ const fs = new MountableFs({ import { Bash, LazyFs, MountableFs, InMemoryFs } from "just-bash"; const lazyFs = new LazyFs({ - // Called when reading a directory + // Called when reading a directory - return entries or null listDir: async (dirPath) => { - const entries = await fetchDirectoryFromAPI(dirPath); - return entries; // [{ name: "file.txt", type: "file" }, ...] or null + // Replace with your API call, e.g.: await fetchDirectoryFromAPI(dirPath) + return [ + { name: "file.txt", type: "file" as const }, + { name: "subdir", type: "directory" as const }, + ]; }, - // Called when reading a file + // Called when reading a file - return content or null loadFile: async (filePath) => { - const content = await fetchFileFromAPI(filePath); - return content; // { content: "...", mode?, mtime? } or null + // Replace with your API call, e.g.: await fetchFileFromAPI(filePath) + return { content: "file content here" }; }, allowWrites: true, // default, writes go to in-memory cache }); From c2f5393ffde773d58b534900b03f31582ef69f99 Mon Sep 17 00:00:00 2001 From: Aron Jones Date: Wed, 28 Jan 2026 21:18:52 -0800 Subject: [PATCH 4/4] Fix LazyFs parent directory tracking for nested paths When writing files to non-existent paths, parent directories created by InMemoryFs were invisible to LazyFs operations. Added markParentsModified() helper to track implicitly created parent directories, and fixed readdir to handle locally-created directories that don't exist in the lazy source. Co-Authored-By: Claude Opus 4.5 --- src/fs/lazy-fs/lazy-fs.test.ts | 83 ++++++++++++++++++++++++++++++++++ src/fs/lazy-fs/lazy-fs.ts | 30 ++++++++++-- 2 files changed, 108 insertions(+), 5 deletions(-) diff --git a/src/fs/lazy-fs/lazy-fs.test.ts b/src/fs/lazy-fs/lazy-fs.test.ts index 2ad36e7d..5eab04c2 100644 --- a/src/fs/lazy-fs/lazy-fs.test.ts +++ b/src/fs/lazy-fs/lazy-fs.test.ts @@ -715,4 +715,87 @@ describe("LazyFs", () => { expect(entries).toContain("local.txt"); }); }); + + describe("parent directory tracking", () => { + it("should make parent directories visible after writeFile to nested path", async () => { + const lazyFs = new LazyFs({ + loadFile: async () => null, + listDir: async (path) => (path === "/" ? [] : null), + }); + + await lazyFs.writeFile("/a/b/c/file.txt", "content"); + + // Parent directories should exist + expect(await lazyFs.exists("/a")).toBe(true); + expect(await lazyFs.exists("/a/b")).toBe(true); + expect(await lazyFs.exists("/a/b/c")).toBe(true); + + // Stats should work + const stat = await lazyFs.stat("/a/b"); + expect(stat.isDirectory).toBe(true); + + // readdir should work + const entries = await lazyFs.readdir("/a"); + expect(entries).toContain("b"); + }); + + it("should make parent directories visible after symlink to nested path", async () => { + const lazyFs = new LazyFs({ + loadFile: async () => null, + listDir: async (path) => (path === "/" ? [] : null), + }); + + await lazyFs.writeFile("/target.txt", "content"); + await lazyFs.symlink("/target.txt", "/a/b/link"); + + // Parent directories should exist + expect(await lazyFs.exists("/a")).toBe(true); + expect(await lazyFs.exists("/a/b")).toBe(true); + + // Stats should work + const stat = await lazyFs.stat("/a"); + expect(stat.isDirectory).toBe(true); + + // readdir should work + const entries = await lazyFs.readdir("/a"); + expect(entries).toContain("b"); + }); + + it("should make parent directories visible after link to nested path", async () => { + const lazyFs = new LazyFs({ + loadFile: async (path) => + path === "/src.txt" ? { content: "content" } : null, + listDir: async (path) => (path === "/" ? [] : null), + }); + + await lazyFs.link("/src.txt", "/a/b/hardlink.txt"); + + // Parent directories should exist + expect(await lazyFs.exists("/a")).toBe(true); + expect(await lazyFs.exists("/a/b")).toBe(true); + + // Stats should work + const stat = await lazyFs.stat("/a"); + expect(stat.isDirectory).toBe(true); + + // readdir should work + const entries = await lazyFs.readdir("/a"); + expect(entries).toContain("b"); + }); + + it("should include nested parent dirs in getAllPaths", async () => { + const lazyFs = new LazyFs({ + loadFile: async () => null, + listDir: async (path) => (path === "/" ? [] : null), + }); + + await lazyFs.writeFile("/x/y/z/file.txt", "content"); + + const paths = lazyFs.getAllPaths(); + expect(paths).toContain("/x"); + expect(paths).toContain("/x/y"); + expect(paths).toContain("/x/y/z"); + expect(paths).toContain("/x/y/z/file.txt"); + }); + }); }); diff --git a/src/fs/lazy-fs/lazy-fs.ts b/src/fs/lazy-fs/lazy-fs.ts index 54fd9165..e5fb3049 100644 --- a/src/fs/lazy-fs/lazy-fs.ts +++ b/src/fs/lazy-fs/lazy-fs.ts @@ -122,6 +122,21 @@ export class LazyFs implements IFileSystem { } } + /** + * Mark all parent directories of a path as modified. + * This ensures parent dirs created by the cache are visible to LazyFs operations. + */ + private markParentsModified(path: string): void { + let parent = this.dirname(path); + while (parent !== "/" && !this.modified.has(parent)) { + this.modified.add(parent); + this.loadedDirs.add(parent); + this.deleted.delete(parent); + this.notExistsAsDir.delete(parent); + parent = this.dirname(parent); + } + } + /** * Ensure a file has been loaded from the lazy loader */ @@ -335,8 +350,9 @@ export class LazyFs implements IFileSystem { // Write to cache await this.cache.writeFile(normalized, content, options); - // Mark as modified + // Mark file and parent dirs as modified this.modified.add(normalized); + this.markParentsModified(normalized); this.deleted.delete(normalized); this.notExistsAsFile.delete(normalized); this.notExistsAsDir.delete(normalized); @@ -521,17 +537,19 @@ export class LazyFs implements IFileSystem { throw new Error(`ENOENT: no such file or directory, scandir '${path}'`); } - // Get entries from loader + // Get entries from loader (may be null for locally-created dirs) const lazyEntries = await this.getDirEntries(normalized); - if (!lazyEntries) { + + // If no lazy entries and directory is not locally modified, it doesn't exist + if (!lazyEntries && !this.modified.has(normalized)) { throw new Error(`ENOENT: no such file or directory, scandir '${path}'`); } // Build result from lazy entries plus any locally added files const entriesMap = new Map(); - // Add lazy entries - for (const entry of lazyEntries) { + // Add lazy entries (if any) + for (const entry of lazyEntries ?? []) { const childPath = normalized === "/" ? `/${entry.name}` : `${normalized}/${entry.name}`; @@ -732,6 +750,7 @@ export class LazyFs implements IFileSystem { await this.cache.symlink(target, normalized); this.modified.add(normalized); + this.markParentsModified(normalized); this.loadedFiles.add(normalized); this.deleted.delete(normalized); this.notExistsAsFile.delete(normalized); @@ -773,6 +792,7 @@ export class LazyFs implements IFileSystem { await this.cache.link(existingNorm, newNorm); this.modified.add(newNorm); + this.markParentsModified(newNorm); this.loadedFiles.add(newNorm); this.deleted.delete(newNorm); this.notExistsAsFile.delete(newNorm);