diff --git a/packages/cli/src/commands/packages/run.test.ts b/packages/cli/src/commands/packages/run.test.ts index a78a260..feb24a8 100644 --- a/packages/cli/src/commands/packages/run.test.ts +++ b/packages/cli/src/commands/packages/run.test.ts @@ -1,6 +1,8 @@ -import { describe, it, expect } from "vitest"; -import { homedir } from "os"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { spawnSync } from "child_process"; +import { homedir, tmpdir } from "os"; import { join } from "path"; +import { mkdirSync, writeFileSync, rmSync, mkdtempSync } from "fs"; import { parsePackageSpec, getCacheDir, @@ -10,6 +12,10 @@ import { substituteEnvVars, getLocalCacheDir, localBundleNeedsExtract, + scanNativeExtensions, + extractDepsRequirements, + getPythonCpythonTag, + installCompatibleDeps, } from "./run.js"; describe("parsePackageSpec", () => { @@ -333,3 +339,157 @@ describe("resolveWorkspace", () => { ); }); }); + +describe("scanNativeExtensions", () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "mpak-test-")); + }); + + afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("returns null for nonexistent directory", () => { + expect(scanNativeExtensions("/nonexistent/path")).toBeNull(); + }); + + it("returns null for empty directory", () => { + expect(scanNativeExtensions(tmpDir)).toBeNull(); + }); + + it("returns null when no native extensions present", () => { + mkdirSync(join(tmpDir, "pydantic"), { recursive: true }); + writeFileSync(join(tmpDir, "pydantic", "__init__.py"), ""); + expect(scanNativeExtensions(tmpDir)).toBeNull(); + }); + + it("extracts cpython tag from .so files", () => { + const subDir = join(tmpDir, "pydantic_core"); + mkdirSync(subDir, { recursive: true }); + writeFileSync( + join(subDir, "_pydantic_core.cpython-313-x86_64-linux-gnu.so"), + "", + ); + expect(scanNativeExtensions(tmpDir)).toBe("cpython313"); + }); + + it("extracts cpython tag from .pyd files", () => { + const subDir = join(tmpDir, "pydantic_core"); + mkdirSync(subDir, { recursive: true }); + writeFileSync( + join(subDir, "_pydantic_core.cpython-312-win_amd64.pyd"), + "", + ); + expect(scanNativeExtensions(tmpDir)).toBe("cpython312"); + }); + + it("returns tag from first match when multiple extensions exist", () => { + const dir1 = join(tmpDir, "aaa_pkg"); + const dir2 = join(tmpDir, "zzz_pkg"); + mkdirSync(dir1, { recursive: true }); + mkdirSync(dir2, { recursive: true }); + writeFileSync(join(dir1, "mod.cpython-310-x86_64-linux-gnu.so"), ""); + writeFileSync(join(dir2, "mod.cpython-313-x86_64-linux-gnu.so"), ""); + const result = scanNativeExtensions(tmpDir); + // Should return one of them (first found via recursive readdir) + expect(result).toMatch(/^cpython3\d+$/); + }); +}); + +describe("extractDepsRequirements", () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "mpak-test-")); + }); + + afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("returns empty array for nonexistent directory", () => { + expect(extractDepsRequirements("/nonexistent/path")).toEqual([]); + }); + + it("returns empty array when no dist-info directories exist", () => { + mkdirSync(join(tmpDir, "pydantic"), { recursive: true }); + expect(extractDepsRequirements(tmpDir)).toEqual([]); + }); + + it("extracts name==version from dist-info directories", () => { + mkdirSync(join(tmpDir, "pydantic_core-2.27.0.dist-info"), { + recursive: true, + }); + mkdirSync(join(tmpDir, "aiohttp-3.9.1.dist-info"), { + recursive: true, + }); + const reqs = extractDepsRequirements(tmpDir); + expect(reqs).toContain("pydantic_core==2.27.0"); + expect(reqs).toContain("aiohttp==3.9.1"); + expect(reqs).toHaveLength(2); + }); + + it("handles versions with multiple dots", () => { + mkdirSync(join(tmpDir, "cryptography-41.0.7.dist-info"), { + recursive: true, + }); + const reqs = extractDepsRequirements(tmpDir); + expect(reqs).toContain("cryptography==41.0.7"); + }); + + it("ignores non-dist-info directories", () => { + mkdirSync(join(tmpDir, "pydantic"), { recursive: true }); + mkdirSync(join(tmpDir, "pydantic_core-2.27.0.dist-info"), { + recursive: true, + }); + const reqs = extractDepsRequirements(tmpDir); + expect(reqs).toEqual(["pydantic_core==2.27.0"]); + }); +}); + +describe("getPythonCpythonTag", () => { + // These tests use the real python on the system + it("returns a valid cpython tag for real python", () => { + // Try python3, fall back to python — skip if neither available + const py3 = spawnSync("python3", ["--version"], { stdio: "pipe" }); + const cmd = py3.status === 0 ? "python3" : "python"; + + const tag = getPythonCpythonTag(cmd); + if (tag === null) { + // No python available — skip + return; + } + expect(tag).toMatch(/^cpython\d\d+$/); + }); + + it("returns null for nonexistent command", () => { + expect(getPythonCpythonTag("nonexistent-python-xyz")).toBeNull(); + }); +}); + +describe("installCompatibleDeps", () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "mpak-test-")); + }); + + afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("throws when both uv and pip fail", () => { + const targetDir = join(tmpDir, "deps"); + mkdirSync(targetDir, { recursive: true }); + + expect(() => + installCompatibleDeps({ + requirements: ["nonexistent-package-xyz==99.99.99"], + targetDir, + pythonCmd: "nonexistent-python-xyz", + }), + ).toThrow("Both uv and pip failed"); + }); +}); diff --git a/packages/cli/src/commands/packages/run.ts b/packages/cli/src/commands/packages/run.ts index 64077a8..ba49125 100644 --- a/packages/cli/src/commands/packages/run.ts +++ b/packages/cli/src/commands/packages/run.ts @@ -4,6 +4,7 @@ import { existsSync, mkdirSync, readFileSync, + readdirSync, writeFileSync, chmodSync, rmSync, @@ -415,6 +416,193 @@ function findPythonCommand(): string { return "python"; } +/** + * Get the user's CPython ABI tag (e.g. "cpython310") + */ +export function getPythonCpythonTag(pythonCmd: string): string | null { + try { + const result = spawnSync( + pythonCmd, + ["-c", "import sys; print(f'cpython{sys.version_info.major}{sys.version_info.minor}')"], + { stdio: "pipe", encoding: "utf8", timeout: 10000 }, + ); + if (result.status === 0 && result.stdout) { + return result.stdout.trim() || null; + } + return null; + } catch { + return null; + } +} + +/** + * Scan deps directory for native extensions and extract the cpython tag from the first match. + * Returns e.g. "cpython313" or null if no native extensions found. + */ +export function scanNativeExtensions(depsDir: string): string | null { + if (!existsSync(depsDir)) return null; + try { + const entries = readdirSync(depsDir, { recursive: true }) as string[]; + for (const entry of entries) { + const match = String(entry).match(/\.cpython-(\d+)[\w-]*\.(so|pyd)$/); + if (match) { + return `cpython${match[1]}`; + } + } + return null; + } catch { + return null; + } +} + +/** + * Extract package requirements from dist-info directories in deps/. + * Returns array of "name==version" strings. + */ +export function extractDepsRequirements(depsDir: string): string[] { + if (!existsSync(depsDir)) return []; + try { + const entries = readdirSync(depsDir); + const requirements: string[] = []; + for (const entry of entries) { + const match = entry.match(/^(.+)-(\d[\w.]*(?:\.\w+)*)\.dist-info$/); + if (match) { + requirements.push(`${match[1]}==${match[2]}`); + } + } + return requirements; + } catch { + return []; + } +} + +/** + * Install compatible deps using uv (preferred) or pip (fallback). + */ +export function installCompatibleDeps(options: { + requirements: string[]; + targetDir: string; + pythonCmd: string; +}): void { + const { requirements, targetDir, pythonCmd } = options; + + const tmpDir = join(homedir(), ".mpak", "tmp"); + mkdirSync(tmpDir, { recursive: true }); + const reqFile = join(tmpDir, `requirements-${Date.now()}.txt`); + + try { + writeFileSync(reqFile, requirements.join("\n")); + + // Try uv first + const uvResult = spawnSync( + "uv", + ["pip", "install", "--target", targetDir, "--python", pythonCmd, "-r", reqFile], + { stdio: "pipe", encoding: "utf8", timeout: 300000 }, + ); + if (uvResult.status === 0) return; + + const uvErr = uvResult.stderr || uvResult.error?.message || "unknown error"; + + // Fall back to pip + const pipResult = spawnSync( + pythonCmd, + ["-m", "pip", "install", "--target", targetDir, "-r", reqFile], + { stdio: "pipe", encoding: "utf8", timeout: 300000 }, + ); + if (pipResult.status === 0) return; + + const pipErr = pipResult.stderr || pipResult.error?.message || "unknown error"; + throw new Error( + `Both uv and pip failed to install deps.\nuv: ${uvErr}\npip: ${pipErr}`, + ); + } finally { + try { + rmSync(reqFile, { force: true }); + } catch { + // ignore cleanup error + } + } +} + +/** + * Ensure Python deps are compatible with the user's Python version. + * Returns the directory to use for PYTHONPATH. + */ +function ensureCompatiblePythonDeps( + depsDir: string, + cacheDir: string, + pythonCmd: string, +): string { + try { + const bundleTag = scanNativeExtensions(depsDir); + if (!bundleTag) return depsDir; // No native extensions — pure Python + + const userTag = getPythonCpythonTag(pythonCmd); + if (!userTag) return depsDir; // Can't detect — best effort + + if (userTag === bundleTag) return depsDir; // Match — happy path + + // Version mismatch — check for cached compatible deps + const versionedDir = join(cacheDir, `.deps-${userTag}`); + if (existsSync(versionedDir)) { + process.stderr.write( + `=> Using cached deps for ${formatTag(userTag)}\n`, + ); + return versionedDir; + } + + // Need to reinstall + const userVersion = formatTag(userTag); + const bundleVersion = formatTag(bundleTag); + process.stderr.write( + `=> Bundle deps built for ${bundleVersion}, you have ${userVersion}\n`, + ); + process.stderr.write( + `=> Installing compatible native extensions...\n`, + ); + + const requirements = extractDepsRequirements(depsDir); + if (requirements.length === 0) return depsDir; + + mkdirSync(versionedDir, { recursive: true }); + try { + installCompatibleDeps({ + requirements, + targetDir: versionedDir, + pythonCmd, + }); + process.stderr.write(`=> Compatible deps installed and cached\n`); + return versionedDir; + } catch (error) { + // Clean up partial install and fall back + try { + rmSync(versionedDir, { recursive: true, force: true }); + } catch { + // ignore + } + const message = error instanceof Error ? error.message : String(error); + process.stderr.write( + `=> Warning: Could not install compatible deps: ${message}\n`, + ); + process.stderr.write(`=> Falling back to bundled deps\n`); + return depsDir; + } + } catch { + return depsDir; // Never crash — fall back to original + } +} + +/** + * Format a cpython tag for display (e.g. "cpython313" -> "Python 3.13") + */ +function formatTag(tag: string): string { + const digits = tag.replace("cpython", ""); + if (digits.length >= 2) { + return `Python ${digits[0]}.${digits.slice(1)}`; + } + return tag; +} + /** * Download a bundle to a file path */ @@ -655,7 +843,9 @@ export async function handleRun( } // Set PYTHONPATH to deps/ directory for dependency resolution - const depsDir = join(cacheDir, "deps"); + // If native extensions were built for a different Python version, reinstall into a versioned cache + const originalDepsDir = join(cacheDir, "deps"); + const depsDir = ensureCompatiblePythonDeps(originalDepsDir, cacheDir, command); const existingPythonPath = process.env["PYTHONPATH"]; env["PYTHONPATH"] = existingPythonPath ? `${depsDir}:${existingPythonPath}`