From cc08dd6876b77b38ff446131d552bfaf8c37e165 Mon Sep 17 00:00:00 2001 From: Filipe Cavalcanti Date: Tue, 20 Jan 2026 10:21:25 -0300 Subject: [PATCH 1/2] improvement: add support for new key value pairs on env file --- ntxbuild/cli.py | 4 ++-- ntxbuild/env_data.py | 51 +++++++++++++++++++++++++++++++++--------- tests/test_env_data.py | 41 +++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 13 deletions(-) create mode 100644 tests/test_env_data.py diff --git a/ntxbuild/cli.py b/ntxbuild/cli.py index 89f82c9..8cb0f8c 100644 --- a/ntxbuild/cli.py +++ b/ntxbuild/cli.py @@ -11,7 +11,7 @@ from .build import BuildTool, nuttx_builder from .config import ConfigManager -from .env_data import clear_ntx_env, load_ntx_env, save_ntx_env +from .env_data import clear_ntx_env, create_base_env_file, load_ntx_env from .setup import download_nuttx_apps_repo, download_nuttx_repo from .utils import NUTTX_APPS_DEFAULT_DIR_NAME, NUTTX_DEFAULT_DIR_NAME, find_nuttx_root @@ -57,7 +57,7 @@ def prepare_env( # This validates the directory structure nuttxspace = find_nuttx_root(current_dir, nuttx_dir, apps_dir) - save_ntx_env(nuttxspace, nuttx_dir, apps_dir, build_tool) + create_base_env_file(nuttxspace, nuttx_dir, apps_dir, build_tool) env = load_ntx_env(nuttxspace) return env["general"] diff --git a/ntxbuild/env_data.py b/ntxbuild/env_data.py index e57d93d..93f4dcc 100644 --- a/ntxbuild/env_data.py +++ b/ntxbuild/env_data.py @@ -1,13 +1,41 @@ import configparser import logging -import os from pathlib import Path # Get logger for this module logger = logging.getLogger("ntxbuild.env_data") -def save_ntx_env( +def append_to_general_section(env_file: Path, key: str, value: str) -> None: + """Append a key-value pair to the general section of the environment file. + + Args: + env_file: Path to the environment file. + key: Key to append. + value: Value to append. + """ + config = configparser.ConfigParser() + config.read(env_file) + config["general"][key] = value + with env_file.open("w", encoding="utf-8") as f: + config.write(f) + + +def remove_from_general_section(env_file: Path, key: str) -> None: + """Remove a key from the general section of the environment file. + + Args: + env_file: Path to the environment file. + key: Key to remove. + """ + config = configparser.ConfigParser() + config.read(env_file) + config["general"].pop(key) + with env_file.open("w", encoding="utf-8") as f: + config.write(f) + + +def create_base_env_file( nuttxspace_path: Path, nuttx_dir: str, apps_dir: str, build_tool: str = "make" ) -> None: """Save environment configuration to an INI file. @@ -20,12 +48,16 @@ def save_ntx_env( """ env_file = nuttxspace_path / ".ntxenv" config = configparser.ConfigParser() - config["general"] = { - "nuttxspace_path": str(nuttxspace_path), - "nuttx_dir": nuttx_dir, - "apps_dir": apps_dir, - "build_tool": build_tool, - } + config.read_dict( + { + "general": { + "nuttxspace_path": str(nuttxspace_path), + "nuttx_dir": nuttx_dir, + "apps_dir": apps_dir, + "build_tool": build_tool, + } + } + ) try: with env_file.open("w", encoding="utf-8") as f: @@ -48,9 +80,6 @@ def load_ntx_env(nuttxspace_path: Path) -> configparser.ConfigParser: Returns None if the file is missing or invalid. """ - # Change into the nuttxspace directory - os.chdir(nuttxspace_path) - env_file = nuttxspace_path / ".ntxenv" if not env_file.exists(): raise FileNotFoundError(f"Environment file does not exist: {env_file}") diff --git a/tests/test_env_data.py b/tests/test_env_data.py new file mode 100644 index 0000000..c34ae84 --- /dev/null +++ b/tests/test_env_data.py @@ -0,0 +1,41 @@ +from ntxbuild.env_data import ( + append_to_general_section, + clear_ntx_env, + create_base_env_file, + load_ntx_env, + remove_from_general_section, +) + + +def test_save_and_load_ntx_env(nuttxspace_path): + """Test save and load ntxenv.""" + create_base_env_file(nuttxspace_path, "nuttx", "apps", "make") + env = load_ntx_env(nuttxspace_path) + assert env["general"]["nuttxspace_path"] == str(nuttxspace_path) + assert env["general"]["nuttx_dir"] == "nuttx" + assert env["general"]["apps_dir"] == "apps" + assert env["general"]["build_tool"] == "make" + clear_ntx_env(nuttxspace_path) + + +def test_append_to_general_section(nuttxspace_path): + """Test append to general section of ntxenv.""" + env_file = nuttxspace_path / ".ntxenv" + create_base_env_file(nuttxspace_path, "nuttx", "apps", "make") + append_to_general_section(env_file, "test_field", "test_value") + env = load_ntx_env(nuttxspace_path) + assert env["general"]["test_field"] == "test_value" + clear_ntx_env(nuttxspace_path) + + +def test_remove_from_general_section(nuttxspace_path): + """Test remove from general section of ntxenv.""" + env_file = nuttxspace_path / ".ntxenv" + create_base_env_file(nuttxspace_path, "nuttx", "apps", "make") + append_to_general_section(env_file, "test_field", "test_value") + env = load_ntx_env(nuttxspace_path) + assert env["general"]["test_field"] == "test_value" + remove_from_general_section(env_file, "test_field") + env = load_ntx_env(nuttxspace_path) + assert "test_field" not in env["general"] + clear_ntx_env(nuttxspace_path) From 2313727d38d854ea3fa4d36fd3c508f661155e82 Mon Sep 17 00:00:00 2001 From: Filipe Cavalcanti Date: Tue, 20 Jan 2026 13:48:46 -0300 Subject: [PATCH 2/2] feature: add support for a toolchain downloader. Supports download toolchain for a few architectures. Automatically adds toolchain to path when building through CLI. Signed-off-by: Filipe Cavalcanti --- .github/workflows/python-package.yml | 4 +- README.md | 88 ++-- docs/source/features.md | 5 + docs/source/ntxbuild.rst | 10 + docs/source/quick_start.md | 50 ++ ntxbuild/__init__.py | 5 +- ntxbuild/cli.py | 81 ++- ntxbuild/toolchains.ini | 15 + ntxbuild/toolchains.py | 589 ++++++++++++++++++++++ ntxbuild/utils.py | 1 + pyproject.toml | 3 + tests/test_cli.py | 20 +- tests/test_toolchains.py | 727 +++++++++++++++++++++++++++ 13 files changed, 1546 insertions(+), 52 deletions(-) create mode 100644 ntxbuild/toolchains.ini create mode 100644 ntxbuild/toolchains.py create mode 100644 tests/test_toolchains.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 03234a1..2880b6d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -32,7 +32,7 @@ jobs: git gperf automake libtool pkg-config build-essential gperf genromfs \ libgmp-dev libmpc-dev libmpfr-dev libisl-dev binutils-dev libelf-dev \ libexpat1-dev gcc-multilib g++-multilib picocom u-boot-tools util-linux \ - kconfig-frontends + kconfig-frontends cmake - name: Install package dependencies run: | python -m pip install --upgrade pip @@ -43,4 +43,4 @@ jobs: pre-commit run --all-files - name: Test with pytest run: | - pytest + pytest -x diff --git a/README.md b/README.md index e258916..f010a16 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,11 @@ **NuttX Build System Assistant** - A Python tool for managing and building NuttX RTOS projects with ease. -ntxbuild is simply a wrapper around the many tools available in the NuttX repository. It wraps around tools +ntxbuild is a wrapper around the many tools available in the NuttX repository. It wraps around utilities such as make, kconfig-tweak, menuconfig and most used bash scripts (such as configure.sh). +Also, it provides different features, such as downloading required toolchains through the CLI. + This tool provides a command line interface that supports NuttX configuration and building, while also providing a Python API that allows you to script your builds. @@ -18,9 +20,8 @@ while also providing a Python API that allows you to script your builds. - **Parallel Builds**: Support for multi-threaded builds with isolated workspaces - **Real-time Output**: Live build progress with proper ANSI escape sequence handling - **Configuration Management**: Kconfig integration for easy system configuration -- **Build Cleanup**: Automated artifact management and cleanup -- **Persistent Settings**: Environment configuration saved in `.ntxenv` files - **Interactive Tools**: Support for curses-based tools like menuconfig +- **Toolchin Support**: Download and use your required toolchain automatically through the CLI ## Requirements @@ -29,36 +30,6 @@ while also providing a Python API that allows you to script your builds. - **Make** and standard build tools required by NuttX RTOS - **CMake** supported but optional -## Installation - -### Using pip - -As an user, you can install this tool using pip: -```bash -pip install ntxbuild -``` - -### From Source -If you are a developer or simply wants to install from source, you can clone -this repository and install using `pip install -e ` - -```bash -git clone -cd ntxbuild -pip install -e . -``` - -Use the `dev` configuration to install development tools and `docs` to install -documentation tools. - -```bash -pip install -e ".[dev]" -``` -```bash -pip install -e ".[docs]" -``` - - ## Quick Start ### 1. Initialize Your NuttX Environment @@ -92,6 +63,57 @@ ntxbuild kconfig --set-value CONFIG_DEBUG=y ntxbuild kconfig --set-str CONFIG_APP_NAME="MyApp" ``` +### Alternative Usage +Alternatively, you can automate your builds using a Python script instead of the CLI. + +```python +from pathlib import Path +from ntxbuild.build import MakeBuilder + +current_dir = Path.cwd() + +# Use the Makefile-based builder +builder = MakeBuilder(current_dir, "nuttx", "nuttx-apps") +# Initialize the board/defconfig +setup_result = builder.initialize("sim", "nsh") + +# Execute the build with 10 parallel jobs +builder.build(parallel=10) + +# You can now clean the environment if needed +builder.distclean() +``` + + +## Installation + +### Using pip + +As an user, you can install this tool using pip: +```bash +pip install ntxbuild +``` + +### From Source +If you are a developer or simply wants to install from source, you can clone +this repository and install using `pip install -e ` + +```bash +git clone +cd ntxbuild +pip install -e . +``` + +Use the `dev` configuration to install development tools and `docs` to install +documentation tools. + +```bash +pip install -e ".[dev]" +``` +```bash +pip install -e ".[docs]" +``` + ## Contributing Contributions are always welcome but will be subject to review and approval. Basic rules: diff --git a/docs/source/features.md b/docs/source/features.md index 7954fd7..d48eb43 100644 --- a/docs/source/features.md +++ b/docs/source/features.md @@ -7,6 +7,11 @@ - Menuconfig available and also kconfig options modification directly from command line - Full Python API support +## Toolchain Management +- The toolchain module provides a quick way to download toolchains for supported architectures. +- Installed on home directory for general use when needed even outside `ntxbuild` CLI. +- Toolchains are organized by NuttX release and default to latest version. + ## CMake Support The use of Makefile is the default state of `ntxbuild`. To build using CMake, pass `--use-cmake` to the `ntxbuild start` command. This will setup the directory to CMake diff --git a/docs/source/ntxbuild.rst b/docs/source/ntxbuild.rst index a95e3fb..d4c972b 100644 --- a/docs/source/ntxbuild.rst +++ b/docs/source/ntxbuild.rst @@ -55,6 +55,16 @@ downloading the NuttX source code. :show-inheritance: :undoc-members: +Toolchains Module +--------------------- + +This contains helper funtions to manage toolchains, such as installing and listing them. + +.. automodule:: ntxbuild.toolchains + :members: + :show-inheritance: + :undoc-members: + Utilities Module --------------------- diff --git a/docs/source/quick_start.md b/docs/source/quick_start.md index 00dc0c6..7ab6d70 100644 --- a/docs/source/quick_start.md +++ b/docs/source/quick_start.md @@ -57,3 +57,53 @@ ntxbuild menuconfig ntxbuild kconfig --set-value CONFIG_DEBUG=y ntxbuild kconfig --set-str CONFIG_APP_NAME="MyApp" ``` + +## Using Python +Alternatively, you can automate your builds using a Python script instead of the CLI. + +```python +from pathlib import Path +from ntxbuild.build import MakeBuilder + +current_dir = Path.cwd() + +# Use the Makefile-based builder +builder = MakeBuilder(current_dir, "nuttx", "nuttx-apps") +# Initialize the board/defconfig +setup_result = builder.initialize("sim", "nsh") + +# Execute the build with 10 parallel jobs +builder.build(parallel=10) + +# You can now clean the environment if needed +builder.distclean() +``` + +## Downloading Toolchains +To visualize currently available toolchains, execute the `toolchain list` command: + +```bash +$ ntxbuild toolchain list +Available toolchains: + - clang-arm-none-eabi + - gcc-aarch64-none-elf + - gcc-arm-none-eabi + - xtensa-esp-elf + - riscv-none-elf +Installed toolchains: + - xtensa-esp-elf +``` + +To install, execute the `toolchain install` command using any of the toolchains from the list above. + +```bash +$ ntxbuild toolchain install gcc-arm-none-eabi +Installing toolchain gcc-arm-none-eabi for NuttX v12.12.0 +โœ… Toolchain gcc-arm-none-eabi installed successfully +Installation directory: /home/fdcavalcanti/ntxenv/toolchains +Note: Toolchains are sourced automatically during build. +``` + +> **_NOTE:_** Toolchains are automatically appended to PATH when building from the CLI. + +> **_NOTE:_** Toolchains are installed to `~/ntxenv/toolchains`. diff --git a/ntxbuild/__init__.py b/ntxbuild/__init__.py index 7b8d34b..2a56cf1 100644 --- a/ntxbuild/__init__.py +++ b/ntxbuild/__init__.py @@ -6,7 +6,6 @@ import logging import sys -from pathlib import Path from . import build, config, utils @@ -17,10 +16,9 @@ def _setup_logging(): """Setup logging configuration for the ntxbuild library.""" # Create logs directory if it doesn't exist - log_dir = Path.home() / ".ntxbuild" / "logs" + log_dir = utils.NTXBUILD_DEFAULT_USER_DIR / "logs" log_dir.mkdir(parents=True, exist_ok=True) - # Configure logging. logging.basicConfig( level=logging.WARNING, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", @@ -29,7 +27,6 @@ def _setup_logging(): ], ) - # Set our library logger level logger = logging.getLogger("ntxbuild") logger.setLevel(logging.WARNING) diff --git a/ntxbuild/cli.py b/ntxbuild/cli.py index 8cb0f8c..d43dfef 100644 --- a/ntxbuild/cli.py +++ b/ntxbuild/cli.py @@ -13,6 +13,7 @@ from .config import ConfigManager from .env_data import clear_ntx_env, create_base_env_file, load_ntx_env from .setup import download_nuttx_apps_repo, download_nuttx_repo +from .toolchains import ManagePath, ToolchainInstaller from .utils import NUTTX_APPS_DEFAULT_DIR_NAME, NUTTX_DEFAULT_DIR_NAME, find_nuttx_root logger = logging.getLogger("ntxbuild.cli") @@ -134,14 +135,12 @@ def main(log_level): @main.command() -def install(): - """Install NuttX and Apps repositories. +def download(): + """Download NuttX and Apps repositories. Downloads the NuttX OS and Apps repositories if they don't already exist in the current directory. If the repositories are already present, this command will verify their existence. - - Exits with code 0 on success. """ current_dir = Path.cwd() click.echo("๐Ÿš€ Downloading NuttX and Apps repositories...") @@ -160,6 +159,75 @@ def install(): sys.exit(0) +@main.group() +def toolchain(): + """Manage NuttX toolchains. + + Provides commands to install and list available toolchains for NuttX builds. + """ + pass + + +@toolchain.command() +@click.argument("toolchain_name", nargs=1, required=True) +@click.argument("nuttx_version", nargs=1, required=False) +def install(toolchain_name, nuttx_version): + """Install a toolchain for a specific NuttX version. + + Downloads and installs a toolchain from the URLs specified in toolchains.ini. + The toolchain will be installed to ~/ntxenv/toolchains//. + + The optional argument NUTTX_VERSION (x.y.z) is the NuttX release version. + Defaults to the latest stable version. + + You can also call 'ntxbuild toolchain list' to see available toolchains. + """ + try: + tinstaller = ToolchainInstaller(toolchain_name, nuttx_version) + except AssertionError: + click.echo( + f"โŒ Toolchain {toolchain_name} not found. " + "Make sure this toolchain is available to download.\n" + "Call 'ntxbuild toolchain list' to see available toolchains." + ) + sys.exit(1) + + click.echo( + f"Installing toolchain {tinstaller.toolchain_name} " + f"for NuttX v{tinstaller.nuttx_version}" + ) + + try: + toolchain_path = tinstaller.install() + except FileExistsError: + click.echo(f"Toolchain {tinstaller.toolchain_name} already installed") + sys.exit(0) + + click.echo(f"โœ… Toolchain {toolchain_name} installed successfully") + click.echo(f"Installation directory: {toolchain_path}") + click.echo("Note: Toolchains are sourced automatically during build.") + sys.exit(0) + + +@toolchain.command() +def list(): + """List available toolchains. + + Displays all toolchains that can be installed for NuttX builds. + """ + toolm = ManagePath() + supported = toolm.supported_toolchains + installed = toolm.installed_toolchains + + click.echo("Available toolchains:") + for toolchain in supported: + click.echo(f" - {toolchain}") + click.echo("Installed toolchains:") + for toolchain in installed: + click.echo(f" - {toolchain}") + sys.exit(0) + + @main.command() @click.option( "--apps-dir", "-a", help="Apps directory", default=NUTTX_APPS_DEFAULT_DIR_NAME @@ -305,6 +373,7 @@ def build(parallel): Exits with code 0 on success, 1 on error, or the build exit code on build failure. """ + ManagePath().add_all_toolchains_to_path() try: builder = get_builder() result = builder.build(parallel) @@ -324,6 +393,7 @@ def distclean(): Exits with code 0 on success. """ click.echo("๐Ÿงน Resetting NuttX environment...") + ManagePath().add_all_toolchains_to_path() builder = get_builder() builder.distclean() clear_ntx_env(builder.nuttxspace_path) @@ -340,6 +410,7 @@ def clean(): Exits with code 0 on success, 1 on error. """ click.echo("๐Ÿงน Cleaning build artifacts...") + ManagePath().add_all_toolchains_to_path() try: builder = get_builder() builder.clean() @@ -370,6 +441,7 @@ def make(ctx): """ command = " ".join(tuple(ctx.args)) click.echo(f"๐Ÿงน Running make {command}") + ManagePath().add_all_toolchains_to_path() builder = get_builder() if builder.build_tool == BuildTool.CMAKE: @@ -397,6 +469,7 @@ def cmake(ctx): """ command = " ".join(tuple(ctx.args)) click.echo(f"๐Ÿงน Running cmake {command}") + ManagePath().add_all_toolchains_to_path() builder = get_builder() if builder.build_tool == BuildTool.MAKE: diff --git a/ntxbuild/toolchains.ini b/ntxbuild/toolchains.ini new file mode 100644 index 0000000..33e0c84 --- /dev/null +++ b/ntxbuild/toolchains.ini @@ -0,0 +1,15 @@ +# Toolchain download links configuration +# This file contains download URLs for toolchains organized by NuttX release version. +# Each section represents a NuttX release version (e.g., [12.0], [12.1], etc.) +# Each entry in a section follows the format: toolchain_name = download_url + +[12.12.0] +clang-arm-none-eabi = https://github.com/ARM-software/LLVM-embedded-toolchain-for-Arm/releases/download/release-17.0.1/LLVMEmbeddedToolchainForArm-17.0.1-Linux-x86_64.tar.xz +gcc-aarch64-none-elf = https://developer.arm.com/-/media/Files/downloads/gnu/13.2.Rel1/binrel/arm-gnu-toolchain-13.2.Rel1-x86_64-aarch64-none-elf.tar.xz +gcc-arm-none-eabi = https://developer.arm.com/-/media/Files/downloads/gnu/13.2.Rel1/binrel/arm-gnu-toolchain-13.2.Rel1-x86_64-arm-none-eabi.tar.xz +riscv-none-elf = https://github.com/xpack-dev-tools/riscv-none-elf-gcc-xpack/releases/download/v13.2.0-2/xpack-riscv-none-elf-gcc-13.2.0-2-linux-x64.tar.gz +xtensa-esp-elf = https://github.com/espressif/crosstool-NG/releases/download/esp-14.2.0_20241119/xtensa-esp-elf-14.2.0_20241119-x86_64-linux-gnu.tar.xz + +[12.11.0] +xtensa-esp-elf = https://github.com/espressif/crosstool-NG/releases/download/esp-14.2.0_20241119/xtensa-esp-elf-14.2.0_20241119-x86_64-linux-gnu.tar.xz +riscv-none-elf = https://github.com/xpack-dev-tools/riscv-none-elf-gcc-xpack/releases/download/v13.2.0-2/xpack-riscv-none-elf-gcc-13.2.0-2-linux-x64.tar.gz diff --git a/ntxbuild/toolchains.py b/ntxbuild/toolchains.py new file mode 100644 index 0000000..9e42653 --- /dev/null +++ b/ntxbuild/toolchains.py @@ -0,0 +1,589 @@ +""" +Toolchain management functions for NuttX builds. + +This module provides classes and functions for installing, managing, and +configuring toolchains for NuttX development. It handles downloading, +extracting, and managing toolchain installations from various sources. +""" + +import configparser +import logging +import os +import shutil +import tarfile +import tempfile +import urllib.request +from dataclasses import dataclass +from enum import Enum +from importlib import resources +from pathlib import Path +from typing import List, Optional + +from packaging import version + +from .utils import NTXBUILD_DEFAULT_USER_DIR + +logger = logging.getLogger("ntxbuild.toolchains") + +DEFAULT_TOOLCHAIN_LOCATION = NTXBUILD_DEFAULT_USER_DIR / "toolchains" + + +@dataclass +class Toolchain: + """Represents a toolchain configuration. + + This dataclass stores information about a toolchain including its name, + download URL, associated NuttX version, and binary directory path. + + Attributes: + name: Name of the toolchain. + url: URL to download the toolchain archive. Defaults to None. + nuttx_version: NuttX version this toolchain is associated with. + Defaults to None. + """ + + name: str + url: Optional[str] = None + nuttx_version: Optional[str] = None + + +class ToolchainName(str, Enum): + """Enumeration of supported toolchain names. + + This enum defines all available toolchain names that can be installed + and managed by the toolchain system. + + Attributes: + CLANG_ARM_NONE_EABI: Clang ARM none EABI toolchain. + GCC_AARCH64_NONE_ELF: GCC AArch64 none ELF toolchain. + GCC_ARM_NONE_EABI: GCC ARM none EABI toolchain. + XTENSA_ESP_ELF: Xtensa ESP ELF toolchain. + RISCV_NONE_ELF: RISC-V none ELF toolchain. + """ + + CLANG_ARM_NONE_EABI = "clang-arm-none-eabi" + GCC_AARCH64_NONE_ELF = "gcc-aarch64-none-elf" + GCC_ARM_NONE_EABI = "gcc-arm-none-eabi" + XTENSA_ESP_ELF = "xtensa-esp-elf" + RISCV_NONE_ELF = "riscv-none-elf" + + def __str__(self): + """Return the string value of the toolchain name. + + Returns: + str: The toolchain name as a string. + """ + return self.value + + +class ToolchainFileParser: + """Parser for toolchain configuration files. + + This class reads and parses toolchain configuration files (toolchains.ini) + to extract toolchain information including names, URLs, and associated + NuttX versions. + """ + + def __init__(self, toolchain_file_path: Optional[Path] = None): + """Initialize the ToolchainFileParser. + + Args: + toolchain_file_path: Path to the toolchain configuration file. + If None, uses the default toolchains.ini from the package. + Defaults to None. + """ + if toolchain_file_path is None: + self.toolchain_file_path = resources.files("ntxbuild").joinpath( + "toolchains.ini" + ) + else: + self.toolchain_file_path = toolchain_file_path + self._toolchains = [] + self._latest_version = None + self._load_toolchains() + + @property + def toolchains(self) -> List[Toolchain]: + """Get the list of parsed toolchains. + + Returns: + List[Toolchain]: List of Toolchain objects parsed from the + configuration file. + """ + if not self._toolchains: + self._load_toolchains() + return self._toolchains + + @property + def latest_version(self) -> Optional[str]: + """Get the latest NuttX version from the toolchain configurations. + + Returns: + Optional[str]: The latest NuttX version string, or None if no + toolchains are available. + """ + if not self._latest_version: + version_list = [ + version.parse(toolchain.nuttx_version) for toolchain in self.toolchains + ] + self._latest_version = max(version_list) + logger.debug(f"Latest NuttX version: {self._latest_version}") + return self._latest_version + + def _load_toolchains(self): + """Load toolchains from the configuration file. + + Parses the toolchains.ini file and populates the internal toolchain + list. Validates that all toolchain names match known ToolchainName + enum values. + + Raises: + ValueError: If no version sections are found in the configuration + file, or if an invalid toolchain name is encountered. + """ + with self.toolchain_file_path.open("r", encoding="utf-8") as f: + config = configparser.ConfigParser() + config.read_file(f) + + # Get all sections (excluding DEFAULT) + sections = [s for s in config.sections() if s != "DEFAULT"] + if not sections: + raise ValueError("No version sections found in toolchains.ini") + + for section in sections: + for toolchain_name, toolchain_url in config[section].items(): + if toolchain_name not in [t.value for t in ToolchainName]: + raise ValueError(f"Invalid toolchain name: {toolchain_name}") + self._toolchains.append( + Toolchain( + name=toolchain_name, url=toolchain_url, nuttx_version=section + ) + ) + + logger.debug(f"Loaded {len(self._toolchains)} toolchains") + for toolchain in self._toolchains: + logger.debug( + f"Toolchain: {toolchain.name}, " + f"NuttX version: {toolchain.nuttx_version}, " + f"URL: {toolchain.url}" + ) + + +class ToolchainInstaller(ToolchainFileParser): + """Installer for NuttX toolchains. + + This class handles downloading and installing toolchains from remote + URLs. It supports various archive formats and manages the installation + process including extraction and directory structure setup. + """ + + def __init__( + self, + toolchain_name: str, + nuttx_version: Optional[str] = None, + toolchain_file_path: Optional[Path] = None, + ): + """Initialize the ToolchainInstaller. + + Args: + toolchain_name: Name of the toolchain to install. Must match + a toolchain name in the configuration file. + nuttx_version: NuttX version to use. If None, uses the latest + available version. Defaults to None. + toolchain_file_path: Path to the toolchain configuration file. + If None, uses the default toolchains.ini from the package. + Defaults to None. + + Raises: + AssertionError: If the toolchain name is not found in the + configuration file, or if the specified NuttX version + is not found. + ValueError: If the toolchain and NuttX version combination + is not found in the configuration file. + """ + super().__init__(toolchain_file_path) + self._toolchain_name = toolchain_name + self._toolchain_install = None + self._nuttx_version = nuttx_version + + assert self._toolchain_name in [ + toolchain.name for toolchain in self.toolchains + ], f"Toolchain: '{self._toolchain_name}' not found in toolchains.ini" + + if self._nuttx_version is None: + self._nuttx_version = str(self.latest_version) + else: + self._nuttx_version = nuttx_version + assert self._nuttx_version in [ + toolchain.nuttx_version for toolchain in self.toolchains + ], f"NuttX version {self._nuttx_version} not found in toolchains.ini" + + for toolchain in self.toolchains: + if ( + toolchain.name == self._toolchain_name + and toolchain.nuttx_version == self._nuttx_version + ): + self._toolchain_install = toolchain + break + else: + raise ValueError( + f"Toolchain '{self._toolchain_name}' NuttX version " + f"{self._nuttx_version} not found in toolchains.ini" + ) + + logger.info( + f"Ready to install toolchain {self._toolchain_name} version " + f"{self._nuttx_version} from {self._toolchain_install.url}" + ) + + def install(self, location: Path = DEFAULT_TOOLCHAIN_LOCATION): + """Install the toolchain to the specified location. + + Downloads the toolchain archive, extracts it, and sets up the + directory structure. The toolchain will be installed in a + subdirectory named after the toolchain. + + Args: + location: Base directory where the toolchain should be installed. + Defaults to DEFAULT_TOOLCHAIN_LOCATION. + + Returns: + Path: The path where the toolchain was installed. + + Raises: + AssertionError: If location is not a Path object. + FileExistsError: If the toolchain directory already exists. + RuntimeError: If downloading or extracting the toolchain fails. + """ + assert isinstance(location, Path), f"Location must be a Path object: {location}" + + location = Path(location).expanduser() + location.mkdir(parents=True, exist_ok=True) + logger.info( + f"Installing toolchain {self._toolchain_name} version " + f"{self._nuttx_version} to {location}" + ) + self._download_and_extract_toolchain(location) + return location + + @property + def toolchain_name(self) -> str: + """Get the toolchain name. + + Returns: + str: The name of the toolchain being installed. + """ + return self._toolchain_name + + @property + def nuttx_version(self) -> str: + """Get the NuttX version. + + Returns: + str: The NuttX version associated with this toolchain installation. + """ + return self._nuttx_version + + def _download_and_extract_toolchain(self, location: Path): + """Download and extract the toolchain archive. + + Downloads the toolchain archive from the configured URL, extracts it + to a temporary directory, and moves it to the final installation + location. Supports multiple archive formats including tar.xz, tar.gz, + tar.bz2, tar, and zip. + + Args: + location: Base directory where the toolchain should be installed. + + Raises: + FileExistsError: If the toolchain directory already exists. + RuntimeError: If downloading or extracting the toolchain fails, + or if the bin directory is not found after installation. + """ + toolchain_dir = location / self._toolchain_name + if toolchain_dir.exists(): + raise FileExistsError( + f"Toolchain {self._toolchain_name} already installed at {toolchain_dir}" + ) + else: + toolchain_dir.mkdir(parents=True, exist_ok=True) + + # Create temporary directory for download + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + archive_path = temp_path / Path(self._toolchain_install.url).name + + # Download the archive + logger.info("Downloading toolchain archive...") + try: + urllib.request.urlretrieve(self._toolchain_install.url, archive_path) + logger.info(f"Downloaded to {archive_path}") + except Exception as e: + raise RuntimeError( + f"Failed to download toolchain from " + f"{self._toolchain_install.url}: {e}" + ) + + # Extract the archive + logger.info("Extracting toolchain archive...") + try: + if archive_path.suffix == ".xz" or archive_path.suffixes[-2:] == [ + ".tar", + ".xz", + ]: + with tarfile.open(archive_path, "r:xz") as tar: + tar.extractall(path=temp_path) + elif archive_path.suffix == ".gz" or archive_path.suffixes[-2:] == [ + ".tar", + ".gz", + ]: + with tarfile.open(archive_path, "r:gz") as tar: + tar.extractall(path=temp_path) + elif archive_path.suffix == ".bz2" or archive_path.suffixes[-2:] == [ + ".tar", + ".bz2", + ]: + with tarfile.open(archive_path, "r:bz2") as tar: + tar.extractall(path=temp_path) + elif archive_path.suffix == ".tar": + with tarfile.open(archive_path, "r") as tar: + tar.extractall(path=temp_path) + elif archive_path.suffix == ".zip": + import zipfile + + with zipfile.ZipFile(archive_path, "r") as zip_ref: + zip_ref.extractall(path=temp_path) + else: + raise RuntimeError( + f"Unsupported archive format: {archive_path.suffix}" + ) + + logger.info(f"Extracted to {temp_path}") + + # Find the extracted directory (usually one top-level directory) + extracted_items = list(temp_path.iterdir()) + if len(extracted_items) == 1 and extracted_items[0].is_dir(): + extracted_dir = extracted_items[0] + else: + # Multiple items or files at root, use temp_path as base + extracted_dir = temp_path + + # Move extracted directory to final location + if toolchain_dir.exists(): + shutil.rmtree(toolchain_dir) + shutil.move(str(extracted_dir), str(toolchain_dir)) + logger.info(f"Toolchain installed to {toolchain_dir}") + + except Exception as e: + # Clean up partial installation + if toolchain_dir.exists(): + shutil.rmtree(toolchain_dir, ignore_errors=True) + raise RuntimeError(f"Failed to extract toolchain: {e}") + + # Verify bin directory exists + if not toolchain_dir.exists(): + raise RuntimeError( + f"Toolchain installed but bin directory not found at {toolchain_dir}" + ) + + logger.info( + f"Toolchain {self._toolchain_name} installed successfully at " + f"{toolchain_dir}" + ) + + return toolchain_dir + + +class ManagePath: + """Manages toolchain PATH environment variable. + + This class handles adding toolchain binary directories to the system PATH + environment variable. It discovers installed toolchains and provides + methods to manage their availability in the current environment. + """ + + def __init__(self, toolchain_location: Path = DEFAULT_TOOLCHAIN_LOCATION): + """Initialize the ManagePath instance. + + Args: + toolchain_location: Base directory where toolchains are installed. + Defaults to DEFAULT_TOOLCHAIN_LOCATION. + """ + self._toolchain_location = toolchain_location + self._supported_toolchains = [str(t) for t in ToolchainName] + self._installed_toolchains = [] + self._load_toolchains() + + @property + def supported_toolchains(self) -> List[str]: + """Get the list of supported toolchain names. + + Returns: + List[str]: List of all supported toolchain names as strings. + """ + return self._supported_toolchains + + @property + def installed_toolchains(self) -> List[ToolchainName]: + """Get the list of installed toolchains. + + Returns: + List[ToolchainName]: List of ToolchainName enum values for + toolchains that are currently installed. + """ + return self._installed_toolchains + + def add_all_toolchains_to_path(self): + """Add all installed toolchains to the PATH environment variable. + + Iterates through all installed toolchains and adds their binary + directories to the system PATH. + """ + for toolchain in self._installed_toolchains: + self.add_toolchain_to_path(str(toolchain)) + + def add_toolchain_to_path(self, toolchain_name: str): + """Add a specific toolchain's binary directory to PATH. + + Finds the toolchain's bin directory and prepends it to the PATH + environment variable. If the directory is already in PATH, it + is not added again. + + Args: + toolchain_name: Name of the toolchain to add to PATH. + + Raises: + AssertionError: If the toolchain is not found in the toolchain + location, or if adding to PATH fails. + ValueError: If the toolchain name does not match any known + toolchain. + RuntimeError: If the toolchain directory structure is invalid + or the bin directory cannot be found. + """ + assert ( + toolchain_name in self._installed_toolchains + ), f"Toolchain {toolchain_name} not found in {self._toolchain_location}" + + toolchain = self._match_toolchain_name(toolchain_name) + toolchain_bin_path = self._parse_toolchain_directory( + self._toolchain_location / str(toolchain) + ) + bin_path_str = str(toolchain_bin_path.resolve()) + + # Current PATH + current_path = os.environ.get("PATH", "") + + # Check if already in PATH (avoid duplicates) + path_dirs = current_path.split(os.pathsep) + if bin_path_str in path_dirs: + logger.debug(f"Toolchain bin directory already in PATH: {bin_path_str}") + return + + # Prepend to PATH + new_path = os.pathsep.join([bin_path_str, current_path]) + os.environ["PATH"] = new_path + assert ( + bin_path_str in os.environ["PATH"] + ), f"Failed to add toolchain bin directory to PATH: {bin_path_str}" + logger.info(f"Added toolchain bin directory to PATH: {bin_path_str}") + + def _match_toolchain_name(self, toolchain_name: str) -> ToolchainName: + """Match a toolchain name string to a ToolchainName enum value. + + Args: + toolchain_name: Toolchain name as a string. + + Returns: + ToolchainName: The corresponding ToolchainName enum value. + + Raises: + ValueError: If the toolchain name does not match any known + toolchain. + """ + for toolchain in ToolchainName: + if toolchain.value == toolchain_name: + return toolchain + raise ValueError(f"Toolchain {toolchain_name} not found") + + def _load_toolchains(self): + """Load installed toolchains from the toolchain location. + + Scans the toolchain location directory and identifies which + toolchains are currently installed by matching directory names + against supported toolchain names. + """ + logger.debug(f"Loading toolchains from {self._toolchain_location}") + logger.debug( + f"Matching to the following toolchains: {self._supported_toolchains}" + ) + + if not self._toolchain_location.exists(): + logger.debug(f"No toolchains installed at {self._toolchain_location}") + return + + for toolchain_dir in self._toolchain_location.iterdir(): + if not toolchain_dir.is_dir(): + continue + + if toolchain_dir.name not in self._supported_toolchains: + continue + + for toolchain in ToolchainName: + if toolchain_dir.name == str(toolchain): + self._installed_toolchains.append(toolchain) + break + + logger.debug( + f"Found {len(self._installed_toolchains)} toolchain(s) in " + f"{self._toolchain_location}" + ) + + def _parse_toolchain_directory(self, toolchain_dir_path: Path) -> Path: + """Parse a toolchain directory to find the bin directory. + + Examines the toolchain directory structure to locate the binary + directory. Handles cases where the toolchain may be nested in a + version-specific subdirectory. + + Args: + toolchain_dir_path: Path to the toolchain directory. + + Returns: + Path: Path to the toolchain's bin directory. + + Raises: + RuntimeError: If the toolchain directory structure is invalid, + if no bin directory is found, or if no executable files + are found in the bin directory. + """ + files = [f for f in list(toolchain_dir_path.iterdir()) if f.is_dir()] + if len(files) == 0: + raise RuntimeError( + "Toolchain directory does not contain any subdirectories: " + f"{toolchain_dir_path}" + ) + if len(files) > 1: + logger.warning( + "Toolchain directory contains multiple versions. " + "Using the first directory available as the toolchain version." + ) + + bin_dir = files[0] / "bin" + if not bin_dir.exists() or not bin_dir.is_dir(): + raise RuntimeError( + f"No 'bin' directory found in toolchain directory: {toolchain_dir_path}" + ) + + # Find at least one executable file in bin_dir + has_executable = False + for file in bin_dir.iterdir(): + if file.is_file() and os.access(file, os.X_OK): + has_executable = True + break + + if not has_executable: + raise RuntimeError( + f"No executable files found in 'bin' directory: {bin_dir}" + ) + + return bin_dir diff --git a/ntxbuild/utils.py b/ntxbuild/utils.py index 0239899..e304550 100644 --- a/ntxbuild/utils.py +++ b/ntxbuild/utils.py @@ -17,6 +17,7 @@ NUTTX_DEFAULT_DIR_NAME = "nuttx" NUTTX_APPS_DEFAULT_DIR_NAME = "nuttx-apps" +NTXBUILD_DEFAULT_USER_DIR = Path.home() / "ntxenv" # Get logger for this module logger = logging.getLogger("ntxbuild.utils") diff --git a/pyproject.toml b/pyproject.toml index 49cdc4e..c78488a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,3 +76,6 @@ local_scheme = "no-local-version" [tool.setuptools.packages.find] include = ["ntxbuild*"] exclude = ["cov_html*", "tests*", "nuttxspace*"] + +[tool.setuptools.package-data] +ntxbuild = ["toolchains.ini"] diff --git a/tests/test_cli.py b/tests/test_cli.py index 2d35bcd..f2231bd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,7 +11,7 @@ clean, cmake, distclean, - install, + download, kconfig, main, make, @@ -137,6 +137,7 @@ def teardown_after_class(self, nuttxspace_path): env_file = nuttxspace_path / ".ntxenv" if env_file.exists(): runner = CliRunner() + runner.invoke(clean, []) runner.invoke(distclean, []) def test_start_with_cmake(self, nuttxspace_path): @@ -180,6 +181,7 @@ def teardown_after_class(self, nuttxspace_path): env_file = nuttxspace_path / ".ntxenv" if env_file.exists(): runner = CliRunner() + runner.invoke(clean, []) runner.invoke(distclean, []) def test_build_with_parallel_jobs(self, nuttxspace_path): @@ -189,8 +191,8 @@ def test_build_with_parallel_jobs(self, nuttxspace_path): # Setup environment first runner = CliRunner() runner.invoke(start, ["sim", "nsh"]) + result = runner.invoke(build, ["-j10"]) - result = runner.invoke(build, ["-j4"]) assert result.exit_code == 0 assert (Path(nuttxspace_path) / "nuttx" / "nuttx").exists() @@ -316,15 +318,15 @@ def test_distclean_without_ntxenv(self, nuttxspace_path): ) -class TestInstall: - """Test suite for the install command.""" +class TestDownload: + """Test suite for the download command.""" def test_install_when_repos_exist(self, nuttxspace_path): - """Test install command when repositories already exist.""" + """Test download command when repositories already exist.""" os.chdir(nuttxspace_path) runner = CliRunner() - result = runner.invoke(install, []) + result = runner.invoke(download, []) assert result.exit_code == 0 assert ( @@ -333,11 +335,11 @@ def test_install_when_repos_exist(self, nuttxspace_path): ) def test_install_verifies_structure(self, nuttxspace_path): - """Test install command verifies directory structure.""" + """Test download command verifies directory structure.""" os.chdir(nuttxspace_path) runner = CliRunner() - result = runner.invoke(install, []) + result = runner.invoke(download, []) assert result.exit_code == 0 # Verify directories exist @@ -532,7 +534,7 @@ def setup_and_teardown_environment(self, nuttxspace_path): env_file = nuttxspace_path / ".ntxenv" if env_file.exists(): runner = CliRunner() - runner.invoke(distclean, []) + runner.invoke(clean, []) def test_cmake_command(self, nuttxspace_path): """Test cmake command with a valid target.""" diff --git a/tests/test_toolchains.py b/tests/test_toolchains.py new file mode 100644 index 0000000..1832df2 --- /dev/null +++ b/tests/test_toolchains.py @@ -0,0 +1,727 @@ +""" +Tests for toolchain management classes. + +This module contains tests for the class-based toolchain API including +ToolchainFileParser, ToolchainInstaller, and ManagePath. +""" + +import os +import shutil +import stat +import tarfile +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest + +from ntxbuild.toolchains import ( + ManagePath, + Toolchain, + ToolchainFileParser, + ToolchainInstaller, + ToolchainName, +) + + +class TestToolchain: + """Tests for Toolchain dataclass.""" + + def test_toolchain_creation_with_all_fields(self): + """Test creating Toolchain with all fields.""" + toolchain = Toolchain( + name="xtensa-esp-elf", + url="https://example.com/toolchain.tar.xz", + nuttx_version="12.12.0", + ) + assert toolchain.name == "xtensa-esp-elf" + assert toolchain.url == "https://example.com/toolchain.tar.xz" + assert toolchain.nuttx_version == "12.12.0" + + def test_toolchain_creation_with_required_field_only(self): + """Test creating Toolchain with only required name field.""" + toolchain = Toolchain(name="xtensa-esp-elf") + assert toolchain.name == "xtensa-esp-elf" + assert toolchain.url is None + assert toolchain.nuttx_version is None + + def test_toolchain_creation_with_optional_fields(self): + """Test creating Toolchain with some optional fields.""" + toolchain = Toolchain( + name="gcc-arm-none-eabi", url="https://example.com/toolchain.tar.xz" + ) + assert toolchain.name == "gcc-arm-none-eabi" + assert toolchain.url == "https://example.com/toolchain.tar.xz" + assert toolchain.nuttx_version is None + + +class TestToolchainName: + """Tests for ToolchainName enum.""" + + def test_toolchain_name_enum_values(self): + """Test that all expected toolchain names exist in enum.""" + assert ToolchainName.CLANG_ARM_NONE_EABI == "clang-arm-none-eabi" + assert ToolchainName.GCC_AARCH64_NONE_ELF == "gcc-aarch64-none-elf" + assert ToolchainName.GCC_ARM_NONE_EABI == "gcc-arm-none-eabi" + assert ToolchainName.XTENSA_ESP_ELF == "xtensa-esp-elf" + assert ToolchainName.RISCV_NONE_ELF == "riscv-none-elf" + + def test_toolchain_name_str_method(self): + """Test that __str__ returns the enum value.""" + assert str(ToolchainName.XTENSA_ESP_ELF) == "xtensa-esp-elf" + assert str(ToolchainName.GCC_ARM_NONE_EABI) == "gcc-arm-none-eabi" + + +class TestToolchainFileParser: + """Tests for ToolchainFileParser class.""" + + def test_toolchain_file_parser_loads_from_package(self): + """Test that ToolchainFileParser loads toolchains.ini from package.""" + parser = ToolchainFileParser() + toolchains = parser.toolchains + + assert len(toolchains) > 0 + assert all(isinstance(t, Toolchain) for t in toolchains) + assert all(t.name for t in toolchains) + assert all(t.url for t in toolchains) + assert all(t.nuttx_version for t in toolchains) + + def test_toolchain_file_parser_with_custom_path(self, tmp_path): + """Test ToolchainFileParser with custom toolchains.ini path.""" + # Create a test toolchains.ini file + toolchains_ini = tmp_path / "toolchains.ini" + toolchains_ini.write_text( + """[12.12.0] +xtensa-esp-elf = https://example.com/xtensa-esp-elf.tar.xz +""" + ) + + parser = ToolchainFileParser(toolchain_file_path=toolchains_ini) + toolchains = parser.toolchains + + assert len(toolchains) == 1 + assert toolchains[0].name == "xtensa-esp-elf" + assert toolchains[0].nuttx_version == "12.12.0" + assert toolchains[0].url == "https://example.com/xtensa-esp-elf.tar.xz" + + def test_toolchain_file_parser_invalid_toolchain_name(self, tmp_path): + """Test ToolchainFileParser raises ValueError for invalid toolchain name.""" + toolchains_ini = tmp_path / "toolchains.ini" + toolchains_ini.write_text( + """[12.12.0] +invalid-toolchain = https://example.com/toolchain.tar.xz +""" + ) + + with pytest.raises(ValueError, match="Invalid toolchain name"): + ToolchainFileParser(toolchain_file_path=toolchains_ini) + + def test_toolchain_file_parser_no_sections(self, tmp_path): + """Test ToolchainFileParser raises ValueError when no version sections found.""" + toolchains_ini = tmp_path / "toolchains.ini" + toolchains_ini.write_text("# Empty file\n") + + with pytest.raises(ValueError, match="No version sections found"): + ToolchainFileParser(toolchain_file_path=toolchains_ini) + + def test_toolchain_file_parser_latest_version(self): + """Test that latest_version property returns the highest version.""" + parser = ToolchainFileParser() + latest = parser.latest_version + + assert latest is not None + # Should be a version object from packaging.version + assert hasattr(latest, "major") + assert hasattr(latest, "minor") + + def test_toolchain_file_parser_latest_version_cached(self): + """Test that latest_version is cached after first access.""" + parser = ToolchainFileParser() + latest1 = parser.latest_version + latest2 = parser.latest_version + + assert latest1 is latest2 # Same object (cached) + + def test_toolchain_file_parser_toolchains_property(self): + """Test that toolchains property returns list of Toolchain objects.""" + parser = ToolchainFileParser() + toolchains = parser.toolchains + + assert isinstance(toolchains, list) + assert len(toolchains) > 0 + for toolchain in toolchains: + assert isinstance(toolchain, Toolchain) + assert toolchain.name in [t.value for t in ToolchainName] + + +class TestToolchainInstaller: + """Tests for ToolchainInstaller class.""" + + # Class-level attributes to store archive paths + archive_dir = None + tar_xz_archive = None + tar_gz_archive = None + zip_archive = None + + @pytest.fixture(scope="class", autouse=True) + def setup_test_archives(self, tmp_path_factory): + """Create test archives once for all tests in this class.""" + # Create a temporary directory for test archives + archive_dir = tmp_path_factory.mktemp("test_archives") + + # Create tar.xz archive + # Goes up to xtensa-esp-elf/bin/test_compiler + tar_xz_path = archive_dir / "test_toolchain.tar.xz" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path_obj = Path(temp_dir) + extracted_dir = temp_path_obj / "xtensa-esp-elf" + extracted_dir.mkdir() + (extracted_dir / "bin").mkdir() + (extracted_dir / "bin" / "test_compiler").write_text( + "#!/bin/bash\necho test\n" + ) + os.chmod(extracted_dir / "bin" / "test_compiler", stat.S_IRWXU) + + with tarfile.open(tar_xz_path, "w:xz") as tar: + tar.add(extracted_dir, arcname="xtensa-esp-elf") + + # Create tar.gz archive + tar_gz_path = archive_dir / "test_toolchain.tar.gz" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path_obj = Path(temp_dir) + extracted_dir = temp_path_obj / "riscv-none-elf" + extracted_dir.mkdir() + (extracted_dir / "bin").mkdir() + (extracted_dir / "bin" / "compiler").write_text("#!/bin/bash\necho test\n") + os.chmod(extracted_dir / "bin" / "compiler", stat.S_IRWXU) + + with tarfile.open(tar_gz_path, "w:gz") as tar: + tar.add(extracted_dir, arcname="riscv-none-elf") + + # Create zip archive + import zipfile + + zip_path = archive_dir / "test_toolchain.zip" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path_obj = Path(temp_dir) + extracted_dir = temp_path_obj / "xtensa-esp-elf" + extracted_dir.mkdir() + (extracted_dir / "bin").mkdir() + compiler_file = extracted_dir / "bin" / "compiler" + compiler_file.write_text("#!/bin/bash\necho test\n") + os.chmod(compiler_file, stat.S_IRWXU) + + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zip_ref: + # Add the directory structure to the zip + for root, dirs, files in os.walk(extracted_dir): + for file in files: + file_path = Path(root) / file + # Use relative path preserve structure + arcname = file_path.relative_to(extracted_dir.parent) + zip_ref.write(file_path, arcname) + + # Store paths as class attributes for use in tests + TestToolchainInstaller.archive_dir = archive_dir + TestToolchainInstaller.tar_xz_archive = tar_xz_path + TestToolchainInstaller.tar_gz_archive = tar_gz_path + TestToolchainInstaller.zip_archive = zip_path + + print(f"Archive directory: {archive_dir}") + print(f"Tar.xz archive: {tar_xz_path}") + print(f"Tar.gz archive: {tar_gz_path}") + print(f"Zip archive: {zip_path}") + + yield + + # Cleanup happens automatically when tmp_path_factory cleans up + + def test_toolchain_installer_init_with_version(self): + """Test ToolchainInstaller initialization with specific version.""" + installer = ToolchainInstaller("xtensa-esp-elf", "12.12.0") + assert installer._toolchain_name == "xtensa-esp-elf" + assert installer._nuttx_version == "12.12.0" + assert installer._toolchain_install is not None + assert installer._toolchain_install.name == "xtensa-esp-elf" + assert installer._toolchain_install.nuttx_version == "12.12.0" + + def test_toolchain_installer_init_without_version(self): + """Test ToolchainInstaller initialization defaults to latest version.""" + parser = ToolchainFileParser() + latest_version = str(parser.latest_version) + + installer = ToolchainInstaller("xtensa-esp-elf") + assert installer._toolchain_name == "xtensa-esp-elf" + assert installer._nuttx_version == latest_version + + def test_toolchain_installer_init_invalid_toolchain(self): + """Test ToolchainInstaller raises AssertionError for invalid toolchain.""" + with pytest.raises(AssertionError, match="not found in toolchains.ini"): + ToolchainInstaller("invalid-toolchain", "12.12.0") + + def test_toolchain_installer_init_invalid_version(self): + """Test ToolchainInstaller raises AssertionError for invalid version.""" + with pytest.raises(AssertionError, match="not found in toolchains.ini"): + ToolchainInstaller("xtensa-esp-elf", "99.99.99") + + def test_toolchain_installer_init_toolchain_not_in_version(self): + """Test ToolchainInstaller raises ValueError when toolchain not in version.""" + # Use a toolchain that exists but not in the specified version + with pytest.raises(AssertionError, match="not found in toolchains.ini"): + ToolchainInstaller("clang-arm-none-eabi", "5.0.0") + + def test_toolchain_installer_install_creates_directory(self, tmp_path, monkeypatch): + """Test that install() works when directory exists.""" + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() # Directory must exist before install() is called + monkeypatch.setattr(Path, "home", lambda: fake_home) + + installer = ToolchainInstaller("xtensa-esp-elf", "12.12.0") + + def mock_urlretrieve(url, filename): + shutil.copy(TestToolchainInstaller.tar_xz_archive, filename) + + with patch( + "ntxbuild.toolchains.urllib.request.urlretrieve", + side_effect=mock_urlretrieve, + ): + result = installer.install(ntxenv_dir) + + assert result == ntxenv_dir + toolchain_dir = ntxenv_dir / "xtensa-esp-elf" + assert toolchain_dir.exists() + assert (toolchain_dir / "bin").exists() + + def test_toolchain_installer_install_already_installed(self, tmp_path, monkeypatch): + """Test that install() returns early if toolchain already installed.""" + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + # Pre-create the toolchain directory + toolchain_dir = ntxenv_dir / "xtensa-esp-elf" + (toolchain_dir / "bin").mkdir(parents=True) + + installer = ToolchainInstaller("xtensa-esp-elf", "12.12.0") + + # Should not download, just return + with pytest.raises(FileExistsError): + installer.install(ntxenv_dir) + + def test_toolchain_installer_install_mocks_url_download( + self, tmp_path, monkeypatch + ): + """Test that install() uses mocked URL download instead of real download.""" + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + installer = ToolchainInstaller("xtensa-esp-elf", "12.12.0") + + download_called = False + + def mock_urlretrieve(url, filename): + nonlocal download_called + download_called = True + shutil.copy(TestToolchainInstaller.tar_xz_archive, filename) + + with patch( + "ntxbuild.toolchains.urllib.request.urlretrieve", + side_effect=mock_urlretrieve, + ): + installer.install(ntxenv_dir) + + assert download_called, "urlretrieve should have been called" + toolchain_dir = ntxenv_dir / "xtensa-esp-elf" / "xtensa-esp-elf" + assert toolchain_dir.exists() + assert (toolchain_dir / "bin").exists() + + def test_toolchain_installer_install_handles_tar_xz(self, tmp_path, monkeypatch): + """Test that install() handles .tar.xz archives correctly.""" + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + installer = ToolchainInstaller("xtensa-esp-elf", "12.12.0") + + def mock_urlretrieve(url, filename): + print(f"Mocking URL retrieve for: {url} to {filename}") + shutil.copy(TestToolchainInstaller.tar_xz_archive, filename) + + with patch( + "ntxbuild.toolchains.urllib.request.urlretrieve", + side_effect=mock_urlretrieve, + ): + installer.install(ntxenv_dir) + + toolchain_dir = ntxenv_dir / "xtensa-esp-elf" / "xtensa-esp-elf" + assert toolchain_dir.exists() + assert (toolchain_dir / "bin").exists() + + def test_toolchain_installer_install_handles_tar_gz(self, tmp_path, monkeypatch): + """Test that install() handles .tar.gz archives correctly.""" + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + # Create a custom toolchains.ini with .tar.gz URL + toolchains_ini = tmp_path / "toolchains.ini" + toolchains_ini.write_text( + """[12.12.0] +riscv-none-elf = https://example.com/riscv-none-elf.tar.gz +""" + ) + + installer = ToolchainInstaller( + "riscv-none-elf", "12.12.0", toolchain_file_path=toolchains_ini + ) + + def mock_urlretrieve(url, filename): + print(f"Mocking URL retrieve for: {url} to {filename}") + shutil.copy(TestToolchainInstaller.tar_gz_archive, filename) + + with patch( + "ntxbuild.toolchains.urllib.request.urlretrieve", + side_effect=mock_urlretrieve, + ): + installer.install(ntxenv_dir) + + toolchain_dir = ntxenv_dir / "riscv-none-elf" / "riscv-none-elf" + assert toolchain_dir.exists() + assert (toolchain_dir / "bin").exists() + + def test_toolchain_installer_install_handles_zip(self, tmp_path, monkeypatch): + """Test that install() handles .zip archives correctly.""" + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + # Create a custom toolchains.ini with .zip URL + toolchains_ini = tmp_path / "toolchains.ini" + toolchains_ini.write_text( + """[12.12.0] +xtensa-esp-elf = https://example.com/xtensa-esp-elf.zip +""" + ) + + installer = ToolchainInstaller( + "xtensa-esp-elf", "12.12.0", toolchain_file_path=toolchains_ini + ) + + def mock_urlretrieve(url, filename): + # Verify the URL ends with .zip (from our custom toolchains.ini) + assert url.endswith(".zip"), f"URL should end with .zip but got: {url}" + # Copy the zip archive to the destination + shutil.copy(TestToolchainInstaller.zip_archive, filename) + # Verify the copied file is a valid zip + import zipfile + + assert zipfile.is_zipfile( + filename + ), f"Copied file is not a valid zip: {filename}" + + with patch( + "ntxbuild.toolchains.urllib.request.urlretrieve", + side_effect=mock_urlretrieve, + ): + installer.install(ntxenv_dir) + + toolchain_dir = ntxenv_dir / "xtensa-esp-elf" / "xtensa-esp-elf" + assert toolchain_dir.exists() + assert (toolchain_dir / "bin").exists() + + def test_toolchain_installer_install_unsupported_format( + self, tmp_path, monkeypatch + ): + """Test that install() raises RuntimeError for unsupported archive format.""" + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + # Create a custom toolchains.ini with unsupported format + toolchains_ini = tmp_path / "toolchains.ini" + toolchains_ini.write_text( + """[12.12.0] +xtensa-esp-elf = https://example.com/xtensa-esp-elf.rar +""" + ) + + installer = ToolchainInstaller( + "xtensa-esp-elf", "12.12.0", toolchain_file_path=toolchains_ini + ) + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path_obj = Path(temp_dir) + archive_path = temp_path_obj / "toolchain.rar" + archive_path.write_text("fake archive") + + def mock_urlretrieve(url, filename): + shutil.copy(archive_path, filename) + + with patch( + "ntxbuild.toolchains.urllib.request.urlretrieve", + side_effect=mock_urlretrieve, + ): + with pytest.raises(RuntimeError, match="Unsupported archive format"): + installer.install(ntxenv_dir) + + def test_toolchain_installer_install_download_failure(self, tmp_path, monkeypatch): + """Test that install() raises RuntimeError when download fails.""" + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + installer = ToolchainInstaller("xtensa-esp-elf", "12.12.0") + + def mock_urlretrieve(url, filename): + raise Exception("Network error") + + with patch( + "ntxbuild.toolchains.urllib.request.urlretrieve", + side_effect=mock_urlretrieve, + ): + with pytest.raises(RuntimeError, match="Failed to download toolchain"): + installer.install(ntxenv_dir) + + def test_toolchain_installer_install_location_not_path(self, tmp_path, monkeypatch): + """Test that install() raises AssertionError when location is not a Path.""" + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + installer = ToolchainInstaller("xtensa-esp-elf", "12.12.0") + + with pytest.raises(AssertionError, match="must be a Path object"): + installer.install("/some/string/path") + + +class TestManagePath: + """Tests for ManagePath class.""" + + def test_manage_path_init_loads_toolchains(self, tmp_path, monkeypatch): + """Test that ManagePath loads installed toolchains on init.""" + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + # Create a toolchain directory structure (as installed by ToolchainInstaller) + # The structure is: ntxenv / toolchain_name / / bin + toolchain_dir = ntxenv_dir / "xtensa-esp-elf" + # Simulate extracted archive structure (version directory inside) + version_dir = toolchain_dir / "xtensa-esp-elf-14.2.0" + bin_dir = version_dir / "bin" + bin_dir.mkdir(parents=True) + compiler = bin_dir / "compiler" + compiler.write_text("#!/bin/bash\necho test\n") + os.chmod(compiler, stat.S_IRWXU) + + manager = ManagePath(toolchain_location=ntxenv_dir) + assert len(manager._installed_toolchains) > 0 + assert ToolchainName.XTENSA_ESP_ELF in manager._installed_toolchains + + def test_manage_path_add_toolchain_to_path(self, tmp_path, monkeypatch): + """Test that add_toolchain_to_path adds toolchain bin to PATH.""" + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + # Create a toolchain directory structure (as installed by ToolchainInstaller) + toolchain_dir = ntxenv_dir / "xtensa-esp-elf" + # Simulate extracted archive structure (version directory inside) + version_dir = toolchain_dir / "xtensa-esp-elf-14.2.0" + bin_dir = version_dir / "bin" + bin_dir.mkdir(parents=True) + compiler = bin_dir / "compiler" + compiler.write_text("#!/bin/bash\necho test\n") + os.chmod(compiler, stat.S_IRWXU) + + original_path = os.environ.get("PATH", "") + + try: + manager = ManagePath(toolchain_location=ntxenv_dir) + manager.add_toolchain_to_path("xtensa-esp-elf") + + current_path = os.environ.get("PATH", "") + assert str(bin_dir.resolve()) in current_path + assert current_path.startswith(str(bin_dir.resolve()) + os.pathsep) + finally: + os.environ["PATH"] = original_path + + def test_manage_path_add_toolchain_to_path_idempotent(self, tmp_path, monkeypatch): + """Test that add_toolchain_to_path is idempotent.""" + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + # Create a toolchain directory structure (as installed by ToolchainInstaller) + toolchain_dir = ntxenv_dir / "xtensa-esp-elf" + version_dir = toolchain_dir / "xtensa-esp-elf-14.2.0" + bin_dir = version_dir / "bin" + bin_dir.mkdir(parents=True) + compiler = bin_dir / "compiler" + compiler.write_text("#!/bin/bash\necho test\n") + os.chmod(compiler, stat.S_IRWXU) + + original_path = os.environ.get("PATH", "") + + try: + manager = ManagePath(toolchain_location=ntxenv_dir) + manager.add_toolchain_to_path("xtensa-esp-elf") + first_path = os.environ.get("PATH", "") + + manager.add_toolchain_to_path("xtensa-esp-elf") + second_path = os.environ.get("PATH", "") + + assert first_path == second_path + assert second_path.count(str(bin_dir.resolve())) == 1 + finally: + os.environ["PATH"] = original_path + + def test_manage_path_add_toolchain_not_installed(self, tmp_path, monkeypatch): + """Test that add_toolchain_to_path raises AssertionError for non-installed + toolchain. + """ + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + manager = ManagePath(toolchain_location=ntxenv_dir) + + with pytest.raises(AssertionError, match="not found in"): + manager.add_toolchain_to_path("xtensa-esp-elf") + + def test_manage_path_parse_toolchain_directory_no_subdirs( + self, tmp_path, monkeypatch + ): + """Test that _parse_toolchain_directory raises RuntimeError when + no subdirectories. + """ + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + # Create toolchain dir with no subdirectories (only files) + toolchain_dir = ntxenv_dir / "xtensa-esp-elf" + toolchain_dir.mkdir() + (toolchain_dir / "file.txt").write_text("test") + + manager = ManagePath(toolchain_location=ntxenv_dir) + manager._installed_toolchains = [ToolchainName.XTENSA_ESP_ELF] + + with pytest.raises(RuntimeError, match="does not contain any subdirectories"): + manager.add_toolchain_to_path("xtensa-esp-elf") + + def test_manage_path_parse_toolchain_directory_no_bin(self, tmp_path, monkeypatch): + """Test that _parse_toolchain_directory raises RuntimeError when no bin + directory. + """ + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + # Create toolchain dir with subdirectory but no bin + toolchain_dir = ntxenv_dir / "xtensa-esp-elf" + version_dir = toolchain_dir / "xtensa-esp-elf-14.2.0" + version_dir.mkdir(parents=True) + + manager = ManagePath(toolchain_location=ntxenv_dir) + manager._installed_toolchains = [ToolchainName.XTENSA_ESP_ELF] + + with pytest.raises(RuntimeError, match="No 'bin' directory found"): + manager.add_toolchain_to_path("xtensa-esp-elf") + + def test_manage_path_parse_toolchain_directory_no_executable( + self, tmp_path, monkeypatch + ): + """Test that _parse_toolchain_directory raises RuntimeError when + no executable files. + """ + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + # Create toolchain dir with bin but no executables + toolchain_dir = ntxenv_dir / "xtensa-esp-elf" + version_dir = toolchain_dir / "xtensa-esp-elf-14.2.0" + bin_dir = version_dir / "bin" + bin_dir.mkdir(parents=True) + (bin_dir / "file.txt").write_text("not executable") + + manager = ManagePath(toolchain_location=ntxenv_dir) + manager._installed_toolchains = [ToolchainName.XTENSA_ESP_ELF] + + with pytest.raises(RuntimeError, match="No executable files found"): + manager.add_toolchain_to_path("xtensa-esp-elf") + + def test_manage_path_match_toolchain_name(self, tmp_path): + """Test that _match_toolchain_name returns correct ToolchainName enum.""" + manager = ManagePath(toolchain_location=tmp_path) + result = manager._match_toolchain_name("xtensa-esp-elf") + assert result == ToolchainName.XTENSA_ESP_ELF + + def test_manage_path_match_toolchain_name_invalid(self, tmp_path): + """Test that _match_toolchain_name raises ValueError for invalid name.""" + manager = ManagePath(toolchain_location=tmp_path) + with pytest.raises(ValueError, match="not found"): + manager._match_toolchain_name("invalid-toolchain") + + def test_manage_path_multiple_toolchains(self, tmp_path, monkeypatch): + """Test that multiple toolchains can be added to PATH.""" + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + ntxenv_dir = fake_home / "ntxenv" + ntxenv_dir.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + + # Create multiple toolchain directories (as installed by ToolchainInstaller) + for toolchain_name in ["xtensa-esp-elf", "gcc-arm-none-eabi"]: + toolchain_dir = ntxenv_dir / toolchain_name + # Simulate extracted archive structure + version_dir = toolchain_dir / f"{toolchain_name}-1.0.0" + bin_dir = version_dir / "bin" + bin_dir.mkdir(parents=True) + compiler = bin_dir / "compiler" + compiler.write_text("#!/bin/bash\necho test\n") + os.chmod(compiler, stat.S_IRWXU) + + original_path = os.environ.get("PATH", "") + + try: + manager = ManagePath(toolchain_location=ntxenv_dir) + manager.add_toolchain_to_path("xtensa-esp-elf") + manager.add_toolchain_to_path("gcc-arm-none-eabi") + + current_path = os.environ.get("PATH", "") + assert "xtensa-esp-elf" in current_path + assert "gcc-arm-none-eabi" in current_path + finally: + os.environ["PATH"] = original_path