[WIP]feat:support isolated python runtime for building

internal/build/build.go

@@ -45,6 +45,7 @@ import (
 	"github.com/goplus/llgo/internal/firmware"
 	"github.com/goplus/llgo/internal/mockable"
 	"github.com/goplus/llgo/internal/packages"
+	"github.com/goplus/llgo/internal/pyenv"
 	"github.com/goplus/llgo/internal/typepatch"
 	"github.com/goplus/llgo/ssa/abi"
 	xenv "github.com/goplus/llgo/xtool/env"
@@ -163,7 +164,6 @@ func Do(args []string, conf *Config) ([]Package, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to setup crosscompile: %w", err)
 	}
-
 	// Update GOOS/GOARCH from export if target was used
 	if conf.Target != "" && export.GOOS != "" {
 		conf.Goos = export.GOOS
@@ -287,6 +287,7 @@ func Do(args []string, conf *Config) ([]Package, error) {
 		crossCompile: export,
 		cTransformer: cabi.NewTransformer(prog, conf.AbiMode),
 	}
+
 	pkgs, err := buildAllPkgs(ctx, initial, verbose)
 	check(err)
 	if mode == ModeGen {
@@ -309,7 +310,6 @@ func Do(args []string, conf *Config) ([]Package, error) {
 
 	global, err := createGlobals(ctx, ctx.prog, pkgs)
 	check(err)
-
 	for _, pkg := range initial {
 		if needLink(pkg, mode) {
 			linkMainPkg(ctx, pkg, allPkgs, global, conf, mode, verbose)
@@ -455,6 +455,32 @@ func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs
 				expdArgs := make([]string, 0, len(altParts))
 				for _, param := range altParts {
 					param = strings.TrimSpace(param)
+					if param == "$(pkg-config --libs python3-embed)" {
+						if err := func() error {
+							pyHome := pyenv.PythonHome()
+							steps := []struct {
+								name string
+								run  func() error
+							}{
+								{"prepare Python cache", func() error { return pyenv.EnsureWithFetch("") }},
+								{"setup Python build env", pyenv.EnsureBuildEnv},
+								{"verify Python", pyenv.Verify},
+								{"fix install_name", func() error { return pyenv.FixLibpythonInstallName(pyHome) }},
+							}
+							for _, s := range steps {
+								if e := s.run(); e != nil {
+									return fmt.Errorf("%s: %w", s.name, e)
+								}
+							}
+							return nil
+						}(); err != nil {
+							panic(fmt.Sprintf("python toolchain init failed: %v\n\tLLGO_CACHE_DIR=%s\n\tPYTHONHOME=%s\n\thint: set LLPYG_PYHOME or check network/permissions",
+								err, env.LLGoCacheDir(), pyenv.PythonHome()))
+						}
+						// if err = pyenv.EnsurePcRpath(pyenv.PythonHome()); err != nil {
+						// 	panic(fmt.Sprintf("failed to inject rpath into python3-embed.pc: %v", err))
+						// }
+					}
 					if strings.ContainsRune(param, '$') {
 						expdArgs = append(expdArgs, xenv.ExpandEnvToArgs(param)...)
 						ctx.nLibdir++
@@ -489,6 +515,18 @@ func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs
 				}
 				aPkg.LinkArgs = append(aPkg.LinkArgs, pkgLinkArgs...)
 			}
+			if kind == cl.PkgPyModule {
+				if name := strings.TrimSpace(param); name != "" {
+					base := strings.Split(name, "@")[0]
+					base = strings.Split(base, "==")[0]
+					if !pyenv.IsStdOrPresent(base) {
+						if err := pyenv.PipInstall(param); err != nil {
+							panic(fmt.Sprintf("pip install failed for '%s': %v\n\tPYTHONHOME=%s\n\thint: ensure pip/network or pin version (e.g. py.numpy==1.26.4)",
+								param, err, pyenv.PythonHome()))
+						}
+					}
+				}
+			}
 		default:
 			err := buildPkg(ctx, aPkg, verbose)
 			if err != nil {
@@ -723,6 +761,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global l
 			}
 		}
 	})
+
 	entryObjFile, err := genMainModuleFile(ctx, llssa.PkgRuntime, pkg, needRuntime, needPyInit)
 	check(err)
 	// defer os.Remove(entryLLFile)
@@ -737,8 +776,24 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global l
 		export, err := exportObject(ctx, pkg.PkgPath+".global", pkg.ExportFile+"-global", []byte(global.String()))
 		check(err)
 		objFiles = append(objFiles, export)
-	}
 
+		addRpath := func(args *[]string, dir string) {
+			if dir == "" {
+				return
+			}
+			flag := "-Wl,-rpath," + dir
+			for _, a := range *args {
+				if a == flag {
+					return
+				}
+			}
+			*args = append(*args, flag)
+		}
+
+		for _, dir := range pyenv.FindPythonRpaths(pyenv.PythonHome()) {
+			addRpath(&linkArgs, dir)
+		}
+	}
 	if IsFullRpathEnabled() {
 		exargs := make([]string, 0, ctx.nLibdir<<1)
 		// Treat every link-time library search path, specified by the -L parameter, as a runtime search path as well.
@@ -751,7 +806,6 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global l
 		}
 		linkArgs = append(linkArgs, exargs...)
 	}
-
 	err = linkObjFiles(ctx, orgApp, objFiles, linkArgs, verbose)
 	check(err)
 

internal/pyenv/fetch.go

@@ -0,0 +1,127 @@
+package pyenv
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func downloadAndExtract(url, dir string) (err error) {
+	if _, err = os.Stat(dir); err == nil {
+		os.RemoveAll(dir)
+	}
+	tempDir := dir + ".temp"
+	os.RemoveAll(tempDir)
+	if err = os.MkdirAll(tempDir, 0755); err != nil {
+		return fmt.Errorf("failed to create temporary directory: %w", err)
+	}
+
+	urlPath := strings.Split(url, "/")
+	filename := urlPath[len(urlPath)-1]
+	localFile := filepath.Join(tempDir, filename)
+	if err = downloadFile(url, localFile); err != nil {
+		return fmt.Errorf("failed to download file: %w", err)
+	}
+	defer os.Remove(localFile)
+
+	if strings.HasSuffix(filename, ".tar.gz") || strings.HasSuffix(filename, ".tgz") {
+		err = extractTarGz(localFile, tempDir)
+	} else {
+		return fmt.Errorf("unsupported archive format: %s", filename)
+	}
+	if err != nil {
+		return fmt.Errorf("failed to extract archive: %w", err)
+	}
+	if err = os.Rename(tempDir, dir); err != nil {
+		return fmt.Errorf("failed to rename directory: %w", err)
+	}
+	return nil
+}
+
+func downloadFile(url, filepath string) error {
+	out, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("bad status: %s", resp.Status)
+	}
+	_, err = io.Copy(out, resp.Body)
+	return err
+}
+
+func extractTarGz(tarGzFile, dest string) error {
+	file, err := os.Open(tarGzFile)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	gzr, err := gzip.NewReader(file)
+	if err != nil {
+		return err
+	}
+	defer gzr.Close()
+	tr := tar.NewReader(gzr)
+	for {
+		header, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		target := filepath.Join(dest, header.Name)
+		if !strings.HasPrefix(target, filepath.Clean(dest)+string(os.PathSeparator)) {
+			return fmt.Errorf("%s: illegal file path", target)
+		}
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, 0755); err != nil {
+				return err
+			}
+		case tar.TypeReg:
+			if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
+				return err
+			}
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(f, tr); err != nil {
+				f.Close()
+				return err
+			}
+			f.Close()
+		case tar.TypeSymlink:
+			if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
+				return err
+			}
+			_ = os.Remove(target)
+			if err := os.Symlink(header.Linkname, target); err != nil {
+				return err
+			}
+		case tar.TypeLink:
+			if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
+				return err
+			}
+			_ = os.Remove(target)
+			linkTarget := filepath.Join(dest, header.Linkname)
+			if err := os.Link(linkTarget, target); err != nil {
+				return err
+			}
+
+		}
+	}
+	return nil
+}

internal/pyenv/llgo-pyenv-design.md

@@ -0,0 +1,110 @@
+# llgo pyenv design
+
+## Goals and Scope
+
+- Goal: Provide a reusable, portable, and bootstrappable CPython runtime (interpreter, standard library, third‑party packages, and link information) for the LLGo build process, and ensure the built artifacts run reliably on target machines without requiring a system Python.
+- Scope: Download and cache, build‑time environment injection, version validation, link info and rpath, runtime initialization assistance, dependency installation, and in‑bundle packaging (bundle).
+
+### Overall Architecture
+
+- Module: `llgo/internal/pyenv`
+  - Role: The Python runtime configurator used during build.
+  - Output: A Python Home (`PYHOME`) consumable by `build`, plus the `.pc` files, `libpython`, headers, standard library, and site‑packages needed for linking and running.
+
+- Directory
+```text
+llgo/internal/pyenv
+├─ pyenv.go
+├─ pybuild.go
+└─ fetch.go
+```
+
+## Core Responsibilities
+
+- Download/Cache
+  - Download and extract CPython into the cache directory: `~/Library/Caches/llgo/python_env/python` (aka `PYHOME`).
+  - API: `EnsureWithFetch("")` (triggered from `build`).
+
+- Build‑time Environment Injection
+  - Set the minimal environment required for building:
+    - Prepend `PYHOME/bin` to `PATH`
+    - Append `PYHOME/lib/pkgconfig` to `PKG_CONFIG_PATH`
+    - Clean up interfering variables (e.g., `PYTHONPATH`)
+  - API: `EnsureBuildEnv()`
+
+- Version and Usability Validation
+  - Enforce Python 3.12 (configurable): in `Verify()`, check `sys.version_info` and run a minimal probe.
+  - API: `Verify()`
+
+- Link Consistency Fix (macOS)
+  - Set `libpython3.*.dylib`’s `LC_ID_DYLIB` to `@rpath/libpython3.*.dylib` to eliminate absolute‑path dependencies (e.g., `/install/...`).
+  - API: `FixLibpythonInstallName(pyHome)`
+
+- Link/Runtime Path (rpath) Recommendation
+  - Provide preferred rpath entries (primarily `PYHOME/lib`; optionally decide whether to fall back to system pkg‑config).
+  - API: `FindPythonRpaths(pyHome)`
+
+- Dependency Installation (site‑packages)
+  - Install third‑party packages into `PYHOME/lib/python3.12/site-packages` with clear guidance and troubleshooting output.
+  - APIs: `InstallPackages`, `PipInstall`
+
+- Runtime Initialization Cooperation (todo)
+  - Provide an embeddable C source (as a string) for `build` to compile into a `.o` that sets `home` (`PyConfig.home`) before `Py_Initialize`, supporting two layouts:
+    - Frameworks bundle layout: `<exe_dir>/Frameworks/libpython...` → `home=<exe_dir>/python`
+    - Regular cache layout: `<PYHOME>/lib/libpython...` → `home=<PYHOME>`
+  - API: `PyInitFromExeDirCSource()`
+
+- In‑bundle Packaging (bundle) (todo)
+  - After a successful build, copy `libpython` and the standard library next to the executable to form an “app‑bundled Python” layout, and fix `install_name`:
+    - `<exe_dir>/Frameworks/libpython3.12.dylib`
+    - `<exe_dir>/python/lib/python3.12/` (including `lib-dynload/` and `site-packages/`)
+  - Suggested API: `BundlePython(app string)` (implemented in `pyenv`, invoked by `build`)
+
+## Integration Points with build (Call Order)
+
+- Location: `llgo/internal/build/build.go` → `linkMainPkg(...)`
+  1) Before expanding external link arguments:
+     - `EnsureWithFetch("")`: prepare the cache
+     - `EnsureBuildEnv()`: inject environment
+     - `Verify()`: enforce 3.12
+     - `FixLibpythonInstallName(pyHome)`: fix `LC_ID_DYLIB` (macOS)
+  2) Aggregate link args and inject rpath:
+     - `FindPythonRpaths(pyenv.PythonHome())`: prioritize `PYHOME/lib`
+     - Add relative rpath: `@executable_path/../Frameworks` (done in `build`)
+  3) Runtime initialization (if embedding is used):
+     - `build` uses `PyInitFromExeDirCSource()` to generate a `.o`, and the main IR calls `__llgo_py_init_from_exedir()` (instead of `Py_Initialize()`)
+  4) After a successful link (final artifact path known):
+     - `pyenv.BundlePython(app)` (optional but recommended)
+
+## Files/Functions (pyenv)
+
+- `PythonHome() string`: returns `PYHOME` (default cache path; optionally honors explicit runtime settings first)
+- `EnsureWithFetch(url string) error`: download/extract Python cache
+- `EnsureBuildEnv() error`: inject build‑time environment
+- `Verify() error`: enforce 3.12 and minimal run check
+- `FixLibpythonInstallName(pyHome string) error`: fix `@rpath` (macOS)
+- `FindPythonRpaths(pyHome string) []string`: return rpath candidates (at least `PYHOME/lib`)
+- `InstallPackages(pkgs ...string) error`, `PipInstall(spec string) error`: install deps to `site-packages`
+- `PyInitFromExeDirCSource() string`: return C source string for `build` to compile into a `.o`
+- `BundlePython(app string) error` (suggested): copy `libpython` and stdlib next to the executable
+
+## Directories and Layouts
+
+- Cache layout (build/install source)
+  - `~/Library/Caches/llgo/python_env/python/`
+    - `bin/python3`
+    - `include/python3.12/`
+    - `lib/libpython3.12.dylib`
+    - `lib/python3.12/` (with `lib-dynload/`, `site-packages/`)
+    - `lib/pkgconfig/python-3.12-embed.pc`
+- In‑bundle layout (todo)
+  - `<exe_dir>/your_app`
+  - `<exe_dir>/Frameworks/libpython3.12.dylib` (install_name: `@rpath/libpython3.12.dylib`)
+  - `<exe_dir>/python/lib/python3.12/` (with `lib-dynload/`, `site-packages/`)
+
+## Version and Platform Strategy
+
+- Version: strictly require Python 3.12 by default (`Verify()`); can be parameterized (3.13+) with corresponding updates to `Fix...`, include paths, and `.pc` names.
+- Platform:
+  - macOS: focus on `install_name`, `@rpath`, `LC_RPATH`, and Frameworks bundle layout.
+  - Others: keep rpath injection and home setting (Linux uses `DT_RPATH`/`RUNPATH`; Windows follows PATH/DLL rules, to be extended).