diff --git a/internal/build/build.go b/internal/build/build.go index 0a66cad6ef..b1860ab718 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -45,6 +45,7 @@ import ( "github.com/goplus/llgo/internal/firmware" "github.com/goplus/llgo/internal/mockable" "github.com/goplus/llgo/internal/packages" + "github.com/goplus/llgo/internal/pyenv" "github.com/goplus/llgo/internal/typepatch" "github.com/goplus/llgo/ssa/abi" xenv "github.com/goplus/llgo/xtool/env" @@ -163,7 +164,6 @@ func Do(args []string, conf *Config) ([]Package, error) { if err != nil { return nil, fmt.Errorf("failed to setup crosscompile: %w", err) } - // Update GOOS/GOARCH from export if target was used if conf.Target != "" && export.GOOS != "" { conf.Goos = export.GOOS @@ -287,6 +287,7 @@ func Do(args []string, conf *Config) ([]Package, error) { crossCompile: export, cTransformer: cabi.NewTransformer(prog, conf.AbiMode), } + pkgs, err := buildAllPkgs(ctx, initial, verbose) check(err) if mode == ModeGen { @@ -309,7 +310,6 @@ func Do(args []string, conf *Config) ([]Package, error) { global, err := createGlobals(ctx, ctx.prog, pkgs) check(err) - for _, pkg := range initial { if needLink(pkg, mode) { linkMainPkg(ctx, pkg, allPkgs, global, conf, mode, verbose) @@ -455,6 +455,32 @@ func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs expdArgs := make([]string, 0, len(altParts)) for _, param := range altParts { param = strings.TrimSpace(param) + if param == "$(pkg-config --libs python3-embed)" { + if err := func() error { + pyHome := pyenv.PythonHome() + steps := []struct { + name string + run func() error + }{ + {"prepare Python cache", func() error { return pyenv.EnsureWithFetch("") }}, + {"setup Python build env", pyenv.EnsureBuildEnv}, + {"verify Python", pyenv.Verify}, + {"fix install_name", func() error { return pyenv.FixLibpythonInstallName(pyHome) }}, + } + for _, s := range steps { + if e := s.run(); e != nil { + return fmt.Errorf("%s: %w", s.name, e) + } + } + return nil + }(); err != nil { + panic(fmt.Sprintf("python toolchain init failed: %v\n\tLLGO_CACHE_DIR=%s\n\tPYTHONHOME=%s\n\thint: set LLPYG_PYHOME or check network/permissions", + err, env.LLGoCacheDir(), pyenv.PythonHome())) + } + // if err = pyenv.EnsurePcRpath(pyenv.PythonHome()); err != nil { + // panic(fmt.Sprintf("failed to inject rpath into python3-embed.pc: %v", err)) + // } + } if strings.ContainsRune(param, '$') { expdArgs = append(expdArgs, xenv.ExpandEnvToArgs(param)...) ctx.nLibdir++ @@ -489,6 +515,18 @@ func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs } aPkg.LinkArgs = append(aPkg.LinkArgs, pkgLinkArgs...) } + if kind == cl.PkgPyModule { + if name := strings.TrimSpace(param); name != "" { + base := strings.Split(name, "@")[0] + base = strings.Split(base, "==")[0] + if !pyenv.IsStdOrPresent(base) { + if err := pyenv.PipInstall(param); err != nil { + panic(fmt.Sprintf("pip install failed for '%s': %v\n\tPYTHONHOME=%s\n\thint: ensure pip/network or pin version (e.g. py.numpy==1.26.4)", + param, err, pyenv.PythonHome())) + } + } + } + } default: err := buildPkg(ctx, aPkg, verbose) if err != nil { @@ -723,6 +761,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global l } } }) + entryObjFile, err := genMainModuleFile(ctx, llssa.PkgRuntime, pkg, needRuntime, needPyInit) check(err) // defer os.Remove(entryLLFile) @@ -737,8 +776,24 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global l export, err := exportObject(ctx, pkg.PkgPath+".global", pkg.ExportFile+"-global", []byte(global.String())) check(err) objFiles = append(objFiles, export) - } + addRpath := func(args *[]string, dir string) { + if dir == "" { + return + } + flag := "-Wl,-rpath," + dir + for _, a := range *args { + if a == flag { + return + } + } + *args = append(*args, flag) + } + + for _, dir := range pyenv.FindPythonRpaths(pyenv.PythonHome()) { + addRpath(&linkArgs, dir) + } + } if IsFullRpathEnabled() { exargs := make([]string, 0, ctx.nLibdir<<1) // Treat every link-time library search path, specified by the -L parameter, as a runtime search path as well. @@ -751,7 +806,6 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global l } linkArgs = append(linkArgs, exargs...) } - err = linkObjFiles(ctx, orgApp, objFiles, linkArgs, verbose) check(err) diff --git a/internal/pyenv/fetch.go b/internal/pyenv/fetch.go new file mode 100644 index 0000000000..36dfe67b49 --- /dev/null +++ b/internal/pyenv/fetch.go @@ -0,0 +1,127 @@ +package pyenv + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" +) + +func downloadAndExtract(url, dir string) (err error) { + if _, err = os.Stat(dir); err == nil { + os.RemoveAll(dir) + } + tempDir := dir + ".temp" + os.RemoveAll(tempDir) + if err = os.MkdirAll(tempDir, 0755); err != nil { + return fmt.Errorf("failed to create temporary directory: %w", err) + } + + urlPath := strings.Split(url, "/") + filename := urlPath[len(urlPath)-1] + localFile := filepath.Join(tempDir, filename) + if err = downloadFile(url, localFile); err != nil { + return fmt.Errorf("failed to download file: %w", err) + } + defer os.Remove(localFile) + + if strings.HasSuffix(filename, ".tar.gz") || strings.HasSuffix(filename, ".tgz") { + err = extractTarGz(localFile, tempDir) + } else { + return fmt.Errorf("unsupported archive format: %s", filename) + } + if err != nil { + return fmt.Errorf("failed to extract archive: %w", err) + } + if err = os.Rename(tempDir, dir); err != nil { + return fmt.Errorf("failed to rename directory: %w", err) + } + return nil +} + +func downloadFile(url, filepath string) error { + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad status: %s", resp.Status) + } + _, err = io.Copy(out, resp.Body) + return err +} + +func extractTarGz(tarGzFile, dest string) error { + file, err := os.Open(tarGzFile) + if err != nil { + return err + } + defer file.Close() + gzr, err := gzip.NewReader(file) + if err != nil { + return err + } + defer gzr.Close() + tr := tar.NewReader(gzr) + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + target := filepath.Join(dest, header.Name) + if !strings.HasPrefix(target, filepath.Clean(dest)+string(os.PathSeparator)) { + return fmt.Errorf("%s: illegal file path", target) + } + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0755); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { + f.Close() + return err + } + f.Close() + case tar.TypeSymlink: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + _ = os.Remove(target) + if err := os.Symlink(header.Linkname, target); err != nil { + return err + } + case tar.TypeLink: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + _ = os.Remove(target) + linkTarget := filepath.Join(dest, header.Linkname) + if err := os.Link(linkTarget, target); err != nil { + return err + } + + } + } + return nil +} diff --git a/internal/pyenv/llgo-pyenv-design.md b/internal/pyenv/llgo-pyenv-design.md new file mode 100644 index 0000000000..8fdbf57a70 --- /dev/null +++ b/internal/pyenv/llgo-pyenv-design.md @@ -0,0 +1,110 @@ +# llgo pyenv design + +## Goals and Scope + +- Goal: Provide a reusable, portable, and bootstrappable CPython runtime (interpreter, standard library, third‑party packages, and link information) for the LLGo build process, and ensure the built artifacts run reliably on target machines without requiring a system Python. +- Scope: Download and cache, build‑time environment injection, version validation, link info and rpath, runtime initialization assistance, dependency installation, and in‑bundle packaging (bundle). + +### Overall Architecture + +- Module: `llgo/internal/pyenv` + - Role: The Python runtime configurator used during build. + - Output: A Python Home (`PYHOME`) consumable by `build`, plus the `.pc` files, `libpython`, headers, standard library, and site‑packages needed for linking and running. + +- Directory +```text +llgo/internal/pyenv +├─ pyenv.go +├─ pybuild.go +└─ fetch.go +``` + +## Core Responsibilities + +- Download/Cache + - Download and extract CPython into the cache directory: `~/Library/Caches/llgo/python_env/python` (aka `PYHOME`). + - API: `EnsureWithFetch("")` (triggered from `build`). + +- Build‑time Environment Injection + - Set the minimal environment required for building: + - Prepend `PYHOME/bin` to `PATH` + - Append `PYHOME/lib/pkgconfig` to `PKG_CONFIG_PATH` + - Clean up interfering variables (e.g., `PYTHONPATH`) + - API: `EnsureBuildEnv()` + +- Version and Usability Validation + - Enforce Python 3.12 (configurable): in `Verify()`, check `sys.version_info` and run a minimal probe. + - API: `Verify()` + +- Link Consistency Fix (macOS) + - Set `libpython3.*.dylib`’s `LC_ID_DYLIB` to `@rpath/libpython3.*.dylib` to eliminate absolute‑path dependencies (e.g., `/install/...`). + - API: `FixLibpythonInstallName(pyHome)` + +- Link/Runtime Path (rpath) Recommendation + - Provide preferred rpath entries (primarily `PYHOME/lib`; optionally decide whether to fall back to system pkg‑config). + - API: `FindPythonRpaths(pyHome)` + +- Dependency Installation (site‑packages) + - Install third‑party packages into `PYHOME/lib/python3.12/site-packages` with clear guidance and troubleshooting output. + - APIs: `InstallPackages`, `PipInstall` + +- Runtime Initialization Cooperation (todo) + - Provide an embeddable C source (as a string) for `build` to compile into a `.o` that sets `home` (`PyConfig.home`) before `Py_Initialize`, supporting two layouts: + - Frameworks bundle layout: `/Frameworks/libpython...` → `home=/python` + - Regular cache layout: `/lib/libpython...` → `home=` + - API: `PyInitFromExeDirCSource()` + +- In‑bundle Packaging (bundle) (todo) + - After a successful build, copy `libpython` and the standard library next to the executable to form an “app‑bundled Python” layout, and fix `install_name`: + - `/Frameworks/libpython3.12.dylib` + - `/python/lib/python3.12/` (including `lib-dynload/` and `site-packages/`) + - Suggested API: `BundlePython(app string)` (implemented in `pyenv`, invoked by `build`) + +## Integration Points with build (Call Order) + +- Location: `llgo/internal/build/build.go` → `linkMainPkg(...)` + 1) Before expanding external link arguments: + - `EnsureWithFetch("")`: prepare the cache + - `EnsureBuildEnv()`: inject environment + - `Verify()`: enforce 3.12 + - `FixLibpythonInstallName(pyHome)`: fix `LC_ID_DYLIB` (macOS) + 2) Aggregate link args and inject rpath: + - `FindPythonRpaths(pyenv.PythonHome())`: prioritize `PYHOME/lib` + - Add relative rpath: `@executable_path/../Frameworks` (done in `build`) + 3) Runtime initialization (if embedding is used): + - `build` uses `PyInitFromExeDirCSource()` to generate a `.o`, and the main IR calls `__llgo_py_init_from_exedir()` (instead of `Py_Initialize()`) + 4) After a successful link (final artifact path known): + - `pyenv.BundlePython(app)` (optional but recommended) + +## Files/Functions (pyenv) + +- `PythonHome() string`: returns `PYHOME` (default cache path; optionally honors explicit runtime settings first) +- `EnsureWithFetch(url string) error`: download/extract Python cache +- `EnsureBuildEnv() error`: inject build‑time environment +- `Verify() error`: enforce 3.12 and minimal run check +- `FixLibpythonInstallName(pyHome string) error`: fix `@rpath` (macOS) +- `FindPythonRpaths(pyHome string) []string`: return rpath candidates (at least `PYHOME/lib`) +- `InstallPackages(pkgs ...string) error`, `PipInstall(spec string) error`: install deps to `site-packages` +- `PyInitFromExeDirCSource() string`: return C source string for `build` to compile into a `.o` +- `BundlePython(app string) error` (suggested): copy `libpython` and stdlib next to the executable + +## Directories and Layouts + +- Cache layout (build/install source) + - `~/Library/Caches/llgo/python_env/python/` + - `bin/python3` + - `include/python3.12/` + - `lib/libpython3.12.dylib` + - `lib/python3.12/` (with `lib-dynload/`, `site-packages/`) + - `lib/pkgconfig/python-3.12-embed.pc` +- In‑bundle layout (todo) + - `/your_app` + - `/Frameworks/libpython3.12.dylib` (install_name: `@rpath/libpython3.12.dylib`) + - `/python/lib/python3.12/` (with `lib-dynload/`, `site-packages/`) + +## Version and Platform Strategy + +- Version: strictly require Python 3.12 by default (`Verify()`); can be parameterized (3.13+) with corresponding updates to `Fix...`, include paths, and `.pc` names. +- Platform: + - macOS: focus on `install_name`, `@rpath`, `LC_RPATH`, and Frameworks bundle layout. + - Others: keep rpath injection and home setting (Linux uses `DT_RPATH`/`RUNPATH`; Windows follows PATH/DLL rules, to be extended). diff --git a/internal/pyenv/pybuild.go b/internal/pyenv/pybuild.go new file mode 100644 index 0000000000..a7929edafe --- /dev/null +++ b/internal/pyenv/pybuild.go @@ -0,0 +1,314 @@ +package pyenv + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strings" + + "github.com/goplus/llgo/internal/env" +) + +// EnsureBuildEnv ensures the Python build environment by: +// - ensuring the cache directory {LLGoCacheDir}/python_env exists (atomic creation) +// - applying PATH/PYTHONHOME/DYLD_LIBRARY_PATH/PKG_CONFIG_PATH +// Priority: use LLPYG_PYHOME if set; otherwise default to the cache path. +func EnsureBuildEnv() error { + if err := Ensure(); err != nil { + return err + } + pyHome := PythonHome() + return applyEnv(pyHome) +} + +func Verify() error { + exe, err := findPythonExec() + if err != nil { + return err + } + // Require Python 3.12 + out, err := exec.Command(exe, "-c", "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}')").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to query Python version: %v", err) + } + ver := strings.TrimSpace(string(out)) + if ver != "3.12" { + return fmt.Errorf("Python 3.12 is required, but detected %s. Please set PYTHONHOME to a 3.12 runtime or install python@3.12.", ver) + } + + cmd := exec.Command(exe, "-c", "import sys; print('OK')") + cmd.Stdout, cmd.Stderr = nil, nil + return cmd.Run() +} + +// PythonHome returns the path that should be used as PYTHONHOME, +func PythonHome() string { + // if v := os.Getenv("PYTHONHOME"); v != "" { + // return v + // } + return filepath.Join(env.LLGoCacheDir(), "python_env", "python") +} + +func findPythonExec() (string, error) { + if p, err := exec.LookPath("python"); err == nil { + return p, nil + } + if p, err := exec.LookPath("python3"); err == nil { + return p, nil + } + return "", exec.ErrNotFound +} +func applyEnv(pyHome string) error { + if pyHome == "" { + return nil + } + bin := filepath.Join(pyHome, "bin") + // lib := filepath.Join(pyHome, "lib") + + // PATH: prepend pyHome/bin if not present + path := os.Getenv("PATH") + parts := strings.Split(path, string(os.PathListSeparator)) + hasBin := false + for _, p := range parts { + if p == bin { + hasBin = true + break + } + } + if !hasBin { + newPath := bin + if path != "" { + newPath += string(os.PathListSeparator) + path + } + if err := os.Setenv("PATH", newPath); err != nil { + return err + } + } + + // PYTHONHOME + if err := os.Setenv("PYTHONHOME", pyHome); err != nil { + return err + } + + // // macOS: DYLD_LIBRARY_PATH append lib if missing + // if runtime.GOOS == "darwin" { + // dyld := os.Getenv("DYLD_LIBRARY_PATH") + // if dyld == "" { + // if err := os.Setenv("DYLD_LIBRARY_PATH", lib); err != nil { + // return err + // } + // } else if !strings.Contains(dyld, lib) { + // if err := os.Setenv("DYLD_LIBRARY_PATH", lib+string(os.PathListSeparator)+dyld); err != nil { + // return err + // } + // } + // } + + // PKG_CONFIG_PATH: add pyHome/lib/pkgconfig + pkgcfg := filepath.Join(pyHome, "lib", "pkgconfig") + pcp := os.Getenv("PKG_CONFIG_PATH") + if pcp == "" { + _ = os.Setenv("PKG_CONFIG_PATH", pkgcfg) + } else { + parts := strings.Split(pcp, string(os.PathListSeparator)) + if !slices.Contains(parts, pkgcfg) { + _ = os.Setenv("PKG_CONFIG_PATH", pkgcfg+string(os.PathListSeparator)+pcp) + } + } + + // Avoid interference from custom PYTHONPATH + _ = os.Unsetenv("PYTHONPATH") + return nil +} + +func InstallPackages(pkgs ...string) error { + pyHome := PythonHome() + if pyHome == "" || len(pkgs) == 0 { + return nil + } + py := filepath.Join(pyHome, "bin", "python3") + site := filepath.Join(pyHome, "lib", "python3.12", "site-packages") + args := []string{"-m", "pip", "install", "--target", site} + args = append(args, pkgs...) + + // pre-run info + fmt.Printf("[pip] Packages : %v\n", pkgs) + fmt.Printf("[pip] Target : %s\n", site) + fmt.Printf("[pip] Interpreter: %s\n", py) + fmt.Printf("[pip] Tip: if the network is slow, set a mirror, e.g.\n") + fmt.Printf(" export PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple\n") + fmt.Printf(" %s -m pip install -i $PIP_INDEX_URL --target %s %v\n", py, site, pkgs) + + cmd := exec.Command(py, args...) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + if err := cmd.Run(); err != nil { + fmt.Fprintf(os.Stderr, "\n[pip] Install failed: %v\n", err) + fmt.Fprintf(os.Stderr, "[pip] Troubleshooting:\n") + fmt.Fprintf(os.Stderr, " - Check network/proxy or use a mirror (set PIP_INDEX_URL)\n") + fmt.Fprintf(os.Stderr, " - Pin a stable version to avoid incompatibilities, e.g. numpy==1.26.4\n") + fmt.Fprintf(os.Stderr, " - For source builds, ensure toolchain and consider --no-binary :all:\n") + fmt.Fprintf(os.Stderr, " - Ensure Python Home has compatible wheels/deps: %s\n", pyHome) + return err + } + + // post-run notice and separation from following output + fmt.Println("\n[pip] Install completed successfully.") + fmt.Printf("[pip] Installed to: %s\n", site) + fmt.Println("------------------------------------------------------------") + fmt.Println() + return nil +} + +func PipInstall(spec string) error { + if spec == "" { + return nil + } + return InstallPackages(spec) +} + +func IsStdOrPresent(mod string) bool { + py := filepath.Join(PythonHome(), "bin", "python3") + code := fmt.Sprintf(`import importlib.util,sys; sys.exit(0 if importlib.util.find_spec(%q) else 1)`, mod) + cmd := exec.Command(py, "-c", code) + return cmd.Run() == nil +} + +func EnsurePcRpath(pyHome string) error { + pc := filepath.Join(pyHome, "lib", "pkgconfig", "python3-embed.pc") + b, err := os.ReadFile(pc) + if err != nil { + return err + } + lines := strings.Split(string(b), "\n") + rpath := "-Wl,-rpath," + filepath.Join(pyHome, "lib") + changed := false + for i, ln := range lines { + if strings.HasPrefix(ln, "Libs:") && !strings.Contains(ln, rpath) { + lines[i] = ln + " " + rpath + changed = true + break + } + } + if !changed { + return nil + } + return os.WriteFile(pc, []byte(strings.Join(lines, "\n")), 0644) +} + +func FixLibpythonInstallName(pyHome string) error { + if runtime.GOOS != "darwin" { + return nil + } + libDir := filepath.Join(pyHome, "lib") + candidates := []string{ + filepath.Join(libDir, "libpython3.12.dylib"), + filepath.Join(libDir, "libpython3.12m.dylib"), + } + var target string + for _, p := range candidates { + if _, err := exec.Command("bash", "-lc", "test -f "+quote(p)).CombinedOutput(); err == nil { + target = p + break + } + } + if target == "" { + return nil + } + if real, err := filepath.EvalSymlinks(target); err == nil && real != "" { + target = real + } + base := filepath.Base(target) + newID := "@rpath/" + base + cmd := exec.Command("install_name_tool", "-id", newID, target) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("install_name_tool -id failed: %v, out=%s", err, string(out)) + } + return nil +} + +func quote(s string) string { return "'" + s + "'" } + +func FindPythonRpaths(pyHome string) []string { + dedup := func(ss []string) []string { + m := make(map[string]struct{}, len(ss)) + var out []string + for _, s := range ss { + if s == "" { + continue + } + if _, ok := m[s]; ok { + continue + } + m[s] = struct{}{} + out = append(out, s) + } + return out + } + hasLibpython := func(dir string) bool { + if dir == "" { + return false + } + ents, err := os.ReadDir(dir) + if err != nil { + return false + } + for _, e := range ents { + name := e.Name() + if strings.HasPrefix(name, "libpython") && strings.HasSuffix(name, ".dylib") { + return true + } + } + return false + } + parseLFlags := func(out string) []string { + var dirs []string + for _, f := range strings.Fields(out) { + if strings.HasPrefix(f, "-L") && len(f) > 2 { + dirs = append(dirs, f[2:]) + } + } + return dirs + } + runPkgConfigDefault := func(args ...string) (string, error) { + cmd := exec.Command("pkg-config", args...) + env := os.Environ() + var filtered []string + for _, kv := range env { + if strings.HasPrefix(kv, "PKG_CONFIG_PATH=") || strings.HasPrefix(kv, "PKG_CONFIG_LIBDIR=") { + continue + } + filtered = append(filtered, kv) + } + cmd.Env = filtered + b, err := cmd.CombinedOutput() + return strings.TrimSpace(string(b)), err + } + + var rpaths []string + + // pyLib := filepath.Join(pyHome, "lib") + // if hasLibpython(pyLib) { + // rpaths = append(rpaths, pyLib) + // } + + names := []string{"python-3.12-embed", "python3-embed", "python-3.12", "python3"} + for _, name := range names { + if out, err := runPkgConfigDefault("--libs", name); err == nil && out != "" { + for _, d := range parseLFlags(out) { + if hasLibpython(d) || d != "" { + rpaths = append(rpaths, d) + } + } + if ld, err := runPkgConfigDefault("--variable=libdir", name); err == nil && ld != "" { + if hasLibpython(ld) || ld != "" { + rpaths = append(rpaths, ld) + } + } + break + } + } + return dedup(rpaths) +} diff --git a/internal/pyenv/pyenv.go b/internal/pyenv/pyenv.go new file mode 100644 index 0000000000..ff4b655abb --- /dev/null +++ b/internal/pyenv/pyenv.go @@ -0,0 +1,111 @@ +package pyenv + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + + "github.com/goplus/llgo/internal/env" +) + +const ( + pyStandaloneTag = "20250808" + pyVersion = "3.12.11" +) + +func defaultPythonURL() string { + base := "https://github.com/astral-sh/python-build-standalone/releases/download/" + pyStandaloneTag + "/" + prefix := "cpython-" + pyVersion + "+" + pyStandaloneTag + "-" + switch runtime.GOOS { + case "darwin": + switch runtime.GOARCH { + case "amd64": + return base + prefix + "x86_64-apple-darwin-install_only.tar.gz" + case "arm64": + return base + prefix + "aarch64-apple-darwin-install_only.tar.gz" + } + case "linux": + panic(fmt.Sprintf("todo: unsupported linux arch %s", runtime.GOARCH)) + // switch runtime.GOARCH { + // case "amd64": + // return base + prefix + "x86_64-unknown-linux-gnu-install_only.tar.gz" + // case "arm64": + // return base + prefix + "aarch64-unknown-linux-gnu-install_only.tar.gz" + // } + default: + panic(fmt.Sprintf("todo: unsupported os %s", runtime.GOOS)) + } + return "" +} + +// Ensure makes sure the Python runtime cache directory exists under +// {LLGoCacheDir()}/python_env using an atomic temp-dir rename pattern. +// It is safe to call concurrently and is idempotent. +func Ensure() error { + root := filepath.Join(env.LLGoCacheDir(), "python_env") + return ensureDirAtomic(root) +} + +// EnsureWithFetch ensures the cache directory exists and, +// if it is empty and url is not empty, downloads and extracts +// assets from the given url into the cache directory. +func EnsureWithFetch(url string) error { + if url == "" { + url = defaultPythonURL() + } + root := filepath.Join(env.LLGoCacheDir(), "python_env") + + if err := ensureDirAtomic(root); err != nil { + return fmt.Errorf("failed to prepare python_env at %q: %w", root, err) + } + + empty, err := isDirEmpty(root) + if err != nil { + return fmt.Errorf("failed to check python_env directory %q: %w", root, err) + } + + if empty { + if url == "" { + return fmt.Errorf("python_env at %q is empty and no download URL provided", root) + } + fmt.Println("downloading python assets from", url) + if err := downloadAndExtract(url, root); err != nil { + return fmt.Errorf("failed to download/extract python assets from %q to %q: %w", url, root, err) + } + } + return nil +} + +func ensureDirAtomic(dir string) error { + if st, err := os.Stat(dir); err == nil && st.IsDir() { + return nil + } + tmp := dir + ".temp" + _ = os.RemoveAll(tmp) + if err := os.MkdirAll(tmp, 0o755); err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + // // Optional marker to indicate successful initialization + // _ = os.WriteFile(filepath.Join(tmp, ".init_ok"), []byte(time.Now().Format(time.RFC3339)), 0o644) + // if err := os.Rename(tmp, dir); err != nil { + // _ = os.RemoveAll(tmp) + // // If another process won the race, treat as success + // if st, err2 := os.Stat(dir); err2 == nil && st.IsDir() { + // return nil + // } + // return fmt.Errorf("rename temp to final: %w", err) + // } + return nil +} + +func isDirEmpty(dir string) (bool, error) { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return true, nil + } + return false, err + } + return len(entries) == 0, nil +}