From 3440b8ff1bc097e638cb62d480685205d70701df Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Thu, 6 Feb 2025 18:07:43 +0100 Subject: [PATCH 01/22] Build system overhaul for Amber rewrite --- builder/nixie/output/nix-wrapped.sh.in | 587 ------------------ builder/default.nix => default.nix | 25 +- flake.lock | 128 +++- flake.nix | 6 +- nix-wrapped.sh.in | 1 - {builder/nixie => nixie}/__init__.py | 0 {builder/nixie => nixie}/__main__.py | 0 {builder/nixie => nixie}/cli/__init__.py | 0 .../nixie => nixie}/cli/commands/add_tool.py | 0 .../nixie => nixie}/cli/commands/for_cmd.py | 0 {builder/nixie => nixie}/cli/commands/init.py | 0 .../nixie => nixie}/cli/commands/update.py | 0 {builder/nixie => nixie}/cli/common.py | 0 {builder/nixie => nixie}/cli/fetchers.py | 0 {builder/nixie => nixie}/nix.py | 0 {builder/nixie => nixie}/nix_channels.py | 0 nixie/output/__init__.py | 0 {builder/nixie => nixie}/output/script.py | 0 {builder/nixie => nixie}/output/tarball.py | 0 nixie/template/__init__.py | 0 {builder/nixie => nixie}/template/apply.py | 0 {builder/nixie => nixie}/template/fetch.py | 0 .../nixie => nixie}/template/nixie_env.py | 0 builder/pyproject.toml => pyproject.toml | 24 +- src/builder.ab | 0 src/cli.ab | 0 src/common.ab | 43 ++ src/main.ab | 27 + src/platform.ab | 46 ++ 29 files changed, 280 insertions(+), 607 deletions(-) delete mode 100644 builder/nixie/output/nix-wrapped.sh.in rename builder/default.nix => default.nix (52%) delete mode 120000 nix-wrapped.sh.in rename {builder/nixie => nixie}/__init__.py (100%) rename {builder/nixie => nixie}/__main__.py (100%) rename {builder/nixie => nixie}/cli/__init__.py (100%) rename {builder/nixie => nixie}/cli/commands/add_tool.py (100%) rename {builder/nixie => nixie}/cli/commands/for_cmd.py (100%) rename {builder/nixie => nixie}/cli/commands/init.py (100%) rename {builder/nixie => nixie}/cli/commands/update.py (100%) rename {builder/nixie => nixie}/cli/common.py (100%) rename {builder/nixie => nixie}/cli/fetchers.py (100%) rename {builder/nixie => nixie}/nix.py (100%) rename {builder/nixie => nixie}/nix_channels.py (100%) create mode 100644 nixie/output/__init__.py rename {builder/nixie => nixie}/output/script.py (100%) rename {builder/nixie => nixie}/output/tarball.py (100%) create mode 100644 nixie/template/__init__.py rename {builder/nixie => nixie}/template/apply.py (100%) rename {builder/nixie => nixie}/template/fetch.py (100%) rename {builder/nixie => nixie}/template/nixie_env.py (100%) rename builder/pyproject.toml => pyproject.toml (64%) create mode 100644 src/builder.ab create mode 100644 src/cli.ab create mode 100644 src/common.ab create mode 100644 src/main.ab create mode 100644 src/platform.ab diff --git a/builder/nixie/output/nix-wrapped.sh.in b/builder/nixie/output/nix-wrapped.sh.in deleted file mode 100644 index bd716f1..0000000 --- a/builder/nixie/output/nix-wrapped.sh.in +++ /dev/null @@ -1,587 +0,0 @@ -#!/usr/bin/env bash -# vim: ts=2 sw=2 fdm=marker fmr={,} fdl=0 -# Nixie (c) Karim Vergnes -# Licensed under GNU GPLv2 -# Required commands: -# - tar (with gzip support) -# - rm, mv, tail, uname, env, chmod, mkdir, tee, cut, tr (coreutils) -# - git -# - kill -# - one of wget, curl or python3 (with SSL support) -# - sh with source and declare -a -# Requirements for building binaries locally: -# - A C and C++ compiler toolchain -# - pkg-config -# - GNU Make -# - flex + bison -# - perl - -[[ "$0" == */* ]] || { >&2 echo "ERROR: This script must be run from an absolute or relative path."; exit 1; } - -SYSTEM="$(uname -s).$(uname -m)" -if [[ "$0" == /* ]] -then - THIS_SCRIPT="$0" -elif readlink "$0" >&/dev/null -then - RL=$(readlink "$0") - if [[ "$RL" == /* ]] - then - THIS_SCRIPT="$RL" - else - THIS_SCRIPT="$PWD/$RL" - fi -else - THIS_SCRIPT="$PWD/$0" -fi -PWD_SAVE=$PWD -REPO_ROOT="$(git -C "${THIS_SCRIPT%/*}" rev-parse --show-toplevel 2>/dev/null || { >&2 echo "WARNING: Failed to find current Git repository, using script parent directory."; echo "${THIS_SCRIPT%/*}"; })" - -if [[ "$SYSTEM" =~ Darwin ]] -then - DLL_EXT="dylib" - USER_STORE="$HOME/Library/Nix" -else - DLL_EXT="so" - USER_STORE="$HOME/.local/share/nix/root" -fi - -if [[ "$XDG_CACHE_HOME" != "" ]] -then - USER_CACHE="$XDG_CACHE_HOME" -elif [[ "$SYSTEM" =~ Darwin ]] -then - USER_CACHE="$HOME/Library/Caches" -else - USER_CACHE="$HOME/.cache" -fi - -##### UTILITY FUNCTIONS ##### - -# Output an error message to stderr then nuke ourselves -_bail() { - tput rmcup - >&2 echo $@ - >&2 echo "This script can be rebuilt using the nixie tool" - kill -ABRT $$ -} - -# Pull a file or directory from the resource tarball linked into this -# script. Useful for semi-offline or completely offline operation. -_untar() { - { - { read -r M - while ! [[ "$M" =~ ^-----BEGIN\ ARCHIVE\ SECTION----- ]] - do read -r M - done - gzip -d -c 2>/dev/null - } < "$THIS_SCRIPT" - if ! [[ $? =~ ^[02]$ ]] - then - # The resource tarball is required since it contains the feature - # attributes for this script. We bail out since there is no - # situation where tar failing would bode well. - _bail "Could not find or decompress resource archive." - fi - } | tar -x "$@" -} - -# Check that a command is available, and if not, add it to the list of missing -# commands to print at the end -declare -a MISSING_CMDS -MISSING_TRIG=0 -_avail() { - which $1 >&/dev/null || { MISSING_TRIG=1; MISSING_CMDS=($MISSING_CMDS "$1"); } -} - -# Print the list of missing commands and return 1, if any commands are missing -_avail_end() { - if [[ $MISSING_TRIG == 1 ]] - then - tput rmcup - >&2 echo "ERROR: The following commands are missing:" - for cmd in "${MISSING_CMDS[@]}" - do - >&2 echo "- $cmd" - done - >&2 echo "Use your distribution's package manager to install them, then try again." - return 1 - else - return 0 - fi -} - -# Best-attempt script for downloading files. We first try wget, then cURL, -# then Python 3. -_dl() { - if which wget >&/dev/null - then - wget "$1" -O"$2" || { rm "$2"; return 1; } - elif which curl >&/dev/null - then - curl -f -L "$1" -o "$2" - elif which python3 >&/dev/null - then - python3 <&2 echo "One of 'wget', 'curl' or 'python3' is required to download files." - >&2 echo "Install one of these then try again." - return 1 - fi -} - - -##### NIX BUILDERS ##### - -# Download script wrapper to retrieve files from the source derivation this -# script was built with. Requires SOURCE_CACHE to be the host name for a -# Cachix-compatible HTTPS host (i.e. with endpoint /serve/xxx-hash/path) -_pull_source() { - [[ -d $2 ]] || - ( - cd "$USER_CACHE" - _untar "sources/$1" - mv "$1" "$2" - ) || ( - cd "$USER_CACHE" - _dl "https://$SOURCE_CACHE/serve/$SOURCE_DERIVATION/$1.tar.gz" "$1.tar.gz" - gzip -d -c "$1.tar.gz" | tar x - mv "$1" "$2" - rm "$1.tar.gz" - ) -} - -# Idem for static Nix binaries -_pull_nix_bin() { - ( - cd "$USER_CACHE" - _untar "$1" - mv "$1" "$2" - ) || ( - _dl "https://$SOURCE_CACHE/serve/$NIX_BINS_DERIVATION/$1" "$2" - ) -} - -_find_or_build_openssl () { - pkg-config libcrypto && return 0 - - echo -ne "\033]0;Building Nix: libcrypto (1/8)\007" - _pull_source "openssl" "$source_root/openssl" - cd "$source_root/openssl" - chmod +x ./config - - # Did you know? The OpenSSL Makefile doesn't include generated headers to - # build deps! No one knows how to write a good Makefile nowadays. - ./config && { - for hdr in $(grep ".*\\.h:" Makefile | cut -f 1 -d :) - do - make $hdr - done - } && - make libcrypto.$DLL_EXT - - cp ./libcrypto.* "$USER_CACHE/nix-lib/" - - export OPENSSL_LIBS="$USER_CACHE/nix-lib" - export OPENSSL_CFLAGS="-I$source_root/openssl/include" -} - -_find_or_build_autoconf () { - libname="$1" - varname="$2" - - echo -ne "\033]0;Building Nix: $libname ($nth/8)\007" - pkg-config $libname && return 0 - - _pull_source "$libname" "$source_root/$libname" - cd "$source_root/$libname" - ./configure && make - - cp ./$libpath/.libs/* "$USER_CACHE/nix-lib/" - - eval "export ${varname}_LIBS=$USER_CACHE/nix-lib" - eval "export ${varname}_CFLAGS=-I$source_root/$libname/$incpfx/include" -} - -_find_or_build_lowdown () { - pkg-config lowdown && return 0 - - echo -ne "\033]0;Building Nix: lowdown (4/8)\007" - _pull_source "lowdown" "$source_root/lowdown" - cd "$source_root/lowdown" - ./configure && make - - if [[ "$SYSTEM" =~ Darwin ]] - then - # macOS' clang doesn't support the GCC-esque --soname, and the library - # output name would be wrong anyway. - cc -shared -o liblowdown.1.dylib *.o - fi - - cp ./liblowdown.* "$USER_CACHE/nix-lib/" - - export LOWDOWN_LIBS="$USER_CACHE/nix-lib" - export LOWDOWN_CFLAGS="-I$source_root/lowdown" -} - -_find_or_build_nlohmann_json () { - pkg-config nlohmann_json && return 0 - - echo -ne "\033]0;Building Nix: nlohmann_json (3/8)\007" - _pull_source "nlohmann_json" "$source_root/nlohmann_json" - - export NLOHMANN_JSON_LIBS="$source_root/nlohmann_json/single_include" - export NLOHMANN_JSON_CFLAGS="-I$source_root/nlohmann_json/single_include" -} - -_find_or_build_boost () { - boost_libs=(atomic chrono container context system thread) - for lb in ${boost_libs[@]} - do - [[ -f /usr/lib/libboost_$lb* ]] || \ - [[ -f /usr/local/lib/libboost_$lb* ]] || boost_not_found=1 && break - [[ -f /usr/include/boost/$lb ]] || \ - [[ -f /usr/local/include/boost/$lb ]] || boost_not_found=1 && break - done - - [[ $boost_not_found == 1 ]] || return 0 - - echo -ne "\033]0;Building Nix: boost (2/8)\007" - _pull_source "boost" "$source_root/boost" - cd "$source_root/boost" - ./bootstrap.sh && ./b2 --with-system --with-thread --with-context --with-container --with-chrono --static - - export BOOST_ROOT="$source_root/boost" -} - -# Check that we have the required dependencies to build Nix locally, then -# do so. -_try_build_nix() { - source_root="$REPO_ROOT/.nixie/sources" - mkdir -p $source_root - - _avail cc - _avail pkg-config - _avail make - _avail flex - _avail bison - _avail perl - _avail_end || return 1; - - _find_or_build_openssl - _find_or_build_boost - _find_or_build_nlohmann_json - _find_or_build_lowdown - libpath= incpfx=c nth=5\ - _find_or_build_autoconf libbrotlicommon LIBBROTLI - libpath=src/libsodium incpfx=src/libsodium nth=6\ - _find_or_build_autoconf libsodium SODIUM - libpath=src incpfx= nth=7\ - _find_or_build_autoconf libeditline EDITLINE - - echo -ne "\033]0;Building Nix (8/8)\007" - _pull_source "nix" "$source_root/nix" - - cd "$source_root/nix" - - # Populate macOS SDK paths - if [[ "$SYSTEM" =~ Darwin ]] - then - macos_sdk="$(xcrun --show-sdk-path)" - [[ -d $macos_sdk ]] || _bail "The macOS SDK from Xcode or CommandLineTools is required to build Nix." - - export LIBCURL_LIBS=$macos_sdk/usr/lib - export LIBCURL_CFLAGS=$macos_sdk/usr/include - export LIBARCHIVE_LIBS=$macos_sdk/usr/lib - export LIBARCHIVE_CFLAGS=$macos_sdk/usr/include - export OPENSSL_LIBS=$macos_sdk/usr/lib - export OPENSSL_CFLAGS=$macos_sdk/usr/include - fi - - ./configure --disable-seccomp-sandboxing \ - --disable-s3 \ - --disable-doc-gen \ - --disable-embedded-sandbox-shell \ - --disable-gc \ - --disable-cpuid \ - && make - - #TODO: determine binary output path and copy to nix-static - mv src/nix/nix "$USER_CACHE/nix-static" -} - -# Try a set of strategies to obtain a statically built Nix binary -_get_nix() { - # Acquiring Nix produces a lot of output, so we use alt-buffer. - tput smcup - echo -ne "\033]0;Building Nix...\007" - - __teardown() { tput rmcup; echo -ne "\033]0;\007"; } - - # And we set a trap to exit alt-buffer on ^C cause we're no savages. - trap "__teardown; exit 1" SIGKILL SIGTERM SIGINT SIGABRT - - if [[ $SYSTEM =~ Darwin ]] && ! [[ -f $USER_CACHE/nix-lib/libfakedir.dylib ]] - then - # Retrieve fakedir - mkdir -p "$USER_CACHE/nix-lib" - _pull_nix_bin "libfakedir.dylib" "$USER_CACHE/nix-lib/libfakedir.dylib" - fi - - # Check if the binary already exists - [[ -f "$USER_CACHE/nix-static" ]] \ - && { __teardown; return 0; } - - # Look in our sources for prebuilt binary - _pull_nix_bin "nix.$SYSTEM" "$USER_CACHE/nix-static" \ - && { __teardown; return 0; } - - # Build Nix locally from source - _try_build_nix 2>&1 | tee nix-build.log \ - && { __teardown; return 0; } - - # Everything failed, bail out - __teardown - return 1 -} - - -##### RUNNER SCRIPTS - -_macos_workaround_nix () { - CMDNAME="$1" - shift 1 - mkdir -p $USER_STORE/nix - - [[ -f "$USER_CACHE/nix-lib/libfakedir.dylib" ]] || _bail "libfakedir.dylib missing, cannot proceed." - - : ${NIX_SSL_CERT_FILE:=/etc/ssl/cert.pem} - export NIX_SSL_CERT_FILE - - # We must ensure fakedir gets propagated into spawned executables as well - # so we unfortunately need to disengage the sandbox entirely. - _NIX_TEST_NO_SANDBOX=1 \ - DYLD_INSERT_LIBRARIES="$USER_CACHE/nix-lib/libfakedir.dylib" \ - DYLD_LIBRARY_PATH="$USER_CACHE/nix-lib" \ - FAKEDIR_PATTERN=/nix \ - FAKEDIR_TARGET=$USER_STORE/nix \ - exec -a "$CMDNAME" "$USER_CACHE/nix-static" "$@" -} - -_catch_nixie_args() { - __help() { : "Show this help and exit." - - >&2 echo "Nix wrapper script, generated by Nixie $NIXIE_VERSION" - >&2 echo - >&2 echo "Available --nixie- options:" - for func in $(declare -F | cut -d' ' -f 3 | grep __) - do - # String between : and ; in function readout is the help tooltip - fdesc="$(declare -f $func | cut -d':' -f 2 -s | head -n 1 | cut -d';' -f 1 | tr -d '"')" - >&2 printf " --nixie-%s\e[30G%s\n" "${func##__}" "$fdesc" - done - exit 0 - } - - __print-config() { : "Print this script's configuration" - - _untar -O features - exit 0 - } - - __extract() { : "Unpack the resources archive in nixie/" - - mkdir -p nixie && cd nixie && _untar - exit 0 - } - - __cleanup() { : "Delete local Nix build files" - - >&2 echo "Removing local Nix channels and build files..." - chmod -R +wx $REPO_ROOT/.nixie && rm -rf $REPO_ROOT/.nixie - >&2 echo "Removing user Nix store..." - chmod -R +wx $USER_STORE && rm -rf $USER_STORE - >&2 echo "Removing retrieved Nix binaries..." - rm -rf $USER_CACHE/nix-static $USER_CACHE/nix-lib - exit 0 - } - - __ignore-system() { : "Do not use system-wide Nix" - - >&2 echo "WARNING: Ignoring system-wide Nix for testing purposes." - >&2 echo "Re-run without the --nixie-ignore-system flag to import the single-user" - >&2 echo "Nix store into the system store." - nosystem=1 - } - - for ((i = 0; i < ${#CMDL_ARGS[@]}; ++i)) - do - arg="${CMDL_ARGS[$i]}" - if [[ $arg =~ ^--nixie-(.*)$ ]] - then - # Drop nixie option from real command line - unset CMDL_ARGS[$i] - if declare -F __${BASH_REMATCH[1]} >&/dev/null - then - __${BASH_REMATCH[1]} - else - >&2 echo "No such option: $arg. Run '$1 --nixie-help' for available options." - exit 1 - fi - fi - done -} - - -##### ENTRY POINT ##### - -# Check for required commands -_avail tar -_avail gzip -_avail uname -_avail_end || exit 1 - -# Load feature attributes of our resource tarball -eval "$(_untar -O features || _bail "The resource archive is missing or malformed.")" - -# Running without a specified sources derivation is not supported, even if sources -# are shipped offline. -[[ "$SOURCE_CACHE" != "" ]] || [[ "$SOURCE_DERIVATION" != "" ]] || [[ "$NIX_BINS_DERIVATION" != "" ]] \ - || _bail "The features file in the resource archive is missing or malformed." - -mkdir -p "$REPO_ROOT/.nixie" - -declare -a EXTRA_ARGS - -declare -a CMDL_ARGS=("$@") - -NIXCMD="nix" - -# Parse --nixie-* args -nosystem=0 -_catch_nixie_args "$0" - -# Unpack builtin Nix channels for channel-oriented commands -if [[ "$0" =~ nix-(shell|build|env)$ ]] -then - if (( $(stat -c %W "$REPO_ROOT/.nixie/channels" 2>/dev/null || echo 0) < $(stat -c %W "$THIS_SCRIPT" ) )) - then - >&2 echo "Unpacking Nix channels, hang tight..." - (cd $REPO_ROOT/.nixie; _untar "channels" 2>/dev/null) - fi - export NIX_PATH="$REPO_ROOT/.nixie/channels:$NIX_PATH" -fi - - -# Emulate nix-shell interpreter behavior -if [[ "$0" =~ nix-shell$ ]] && [[ -f "$1" ]] && ! [[ "$1" =~ .nix$ ]] -then - { - read - IFS=' ' read -a nix_shell_args - - # There is a possibility that argv[1] is a Nix file. - # Bail if there's no second line shebang. - [[ "${nix_shell_args[0]}" =~ ^#! ]] || break - - nix_shell_args=("${nix_shell_args[@]:1}") - [[ "${nix_shell_args[0]}" == "nix-shell" ]] && nix_shell_args=("${nix_shell_args[@]:1}") - - for i in "${!nix_shell_args[@]}" - do - [[ "${nix_shell_args[$i]}" == "-i" ]] && { - nix_shell_args[$i]="--command" - nix_shell_args[$((i+1))]="${nix_shell_args[$((i+1))]} $*" - } && break - done - - # Overwrite shell arguments with real interpreter - CMDL_ARGS=("${nix_shell_args[@]}") - } < "$1" -fi - -# Command substitution feature -if [[ "$0" =~ nix$ ]] || [[ "$0" =~ nix-(shell|build|env|channel|hash|instantiate|store|collect-garbage)$ ]] -then - NIXCMD="$0" -elif [[ -f "$REPO_ROOT/flake.nix" ]] -then - # Try to run named command from flake develop - NIXCMD="nix" - CMDL_ARGS=("develop" "$REPO_ROOT" "-c" "${0##*/}" "$@") -elif [[ -f "$REPO_ROOT/shell.nix" ]] -then - # Try to run named command from shell - NIXCMD="nix-shell" - CMDL_ARGS=("$REPO_ROOT/shell.nix" "--command" "${0##*/}$(printf ' %q' "$@")") -fi - -# Check for alternate OpenSSL/LibreSSL certificate paths (fixes #6) -[[ -f "/etc/pki/tls/certs/ca-bundle.crt" ]] && : ${NIX_SSL_CERT_FILE:="/etc/pki/tls/certs/ca-bundle.crt"} -[[ "$NIX_SSL_CERT_FILE" != "" ]] && export NIX_SSL_CERT_FILE - - -# Apply experimental features if listed -[[ "$EXTRA_FEATURES" != "" ]] && EXTRA_ARGS+=("--extra-experimental-features" "$EXTRA_FEATURES") - -# Apply extra substituters and their signing keys (e.g. cachix) if listed -[[ "$EXTRA_SUBSTITUTERS" != "" ]] && EXTRA_ARGS+=("--extra-substituters" "$EXTRA_SUBSTITUTERS") && nix_daemon_warn=1 -[[ "$EXTRA_TRUSTED_PUBLIC_KEYS" != "" ]] && EXTRA_ARGS+=("--extra-trusted-public-keys" "$EXTRA_TRUSTED_PUBLIC_KEYS") && nix_daemon_warn=1 - - -if [[ $nix_daemon_warn == 1 ]] && \ - pgrep nix-daemon >/dev/null 2>&1 && \ - ! grep "trusted-users\s*=.*$(whoami 2>/dev/null)" /etc/nix/nix.conf >/dev/null 2>&1 -then - >&2 echo "This nix wrapper script specifies additional binary caches," - >&2 echo "but you are running on a multi-user install as an untrusted user." - >&2 echo "Cache substitution may not work until you add yourself to" - >&2 echo "the trusted-users entry in /etc/nix/nix.conf." -fi - -if [[ $nosystem != 1 ]] && [[ -d /nix/store ]] && which nix >&/dev/null -then - # Here, we check that the user just installed Nix proper on their system, - # so we migrate paths away from their home to avoid duplicates. - if [[ -d $USER_STORE/nix/store ]] - then - >&2 echo "Migrating Nix store to system-wide install..." - nix copy --from $USER_STORE --all --no-check-sigs &&\ - chmod -R +wx $USER_STORE && rm -rf $USER_STORE - fi - - exec -a "$NIXCMD" nix "${EXTRA_ARGS[@]}" "${CMDL_ARGS[@]}" -else - if _get_nix && chmod +x "$USER_CACHE/nix-static" - then - # This is required if the Nix binary was built locally. - # It is highly unlikely that the user has static copies of all the - # libraries required by Nix, so it's easier to build a shared binary. - # After all, this binary isn't meant to leave its host machine. - export LD_LIBRARY_PATH="$USER_CACHE/nix-lib:$LD_LIBRARY_PATH" - - if [[ "$SYSTEM" =~ Darwin ]] - then - # wow apple thanks - _macos_workaround_nix "$NIXCMD" "${EXTRA_ARGS[@]}" "${CMDL_ARGS[@]}" - else - # Workaround for ascendant symlinks - mkdir -p $HOME/.local/share/nix/root - exec -a "$NIXCMD" "$USER_CACHE/nix-static" \ - --store "$(readlink -f $HOME/.local/share/nix/root)" \ - "${EXTRA_ARGS[@]}" "${CMDL_ARGS[@]}" - fi - else - >&2 echo "Failed to obtain Nix. Check your internet connection." - exit 1 - fi -fi - -# Prevent overrun into resource tarball -exit 1 -cat < +// Licensed under GNU GPLv2 +// Common utility functions + +import { echo_error } from "std/env" + +let SELF = trust $echo \$0$ + +pub fun bail(message: Text): Null +{ + trust $tput rmcup$ + + echo_error(message, 0) + echo_error("This script can be rebuilt using the nixie tool.",1) + trust $kill -ABRT \$\$$ +} + +pub fun untar(member: Text): Text +{ + return "SOURCE_CACHE=xxx" +} + +pub fun check_deps(deps: []): Bool +{ + let missing = [ Text ] + + for dep in deps { + silent $which {dep}$ failed { + missing += [ dep ] + } + } + + if "{missing}" != "" { + echo_error("The following commands are missing:", 0) + for cmd in missing { + echo_error("- {cmd}", 0) + } + echo_error("Use your distribution's package manager to install them, then try again.", 0) + return false + } + + return true +} diff --git a/src/main.ab b/src/main.ab new file mode 100644 index 0000000..be47660 --- /dev/null +++ b/src/main.ab @@ -0,0 +1,27 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Entry point for script logic + +import { env_var_get } from "std/env" + +import { bail, untar, check_deps } from "./common.ab" + +let SOURCE_CACHE = "" +let SOURCE_DERIVATION = "" +let BINARIES_DERIVATION = "" + +check_deps(["tar", "gzip", "uname"]) + +fun load_features(): Null +{ + $eval {untar("features")}$ failed { + bail("The resource archive is missing or malformed.") + } + + SOURCE_CACHE = trust env_var_get("SOURCE_CACHE") + SOURCE_DERIVATION = trust env_var_get("SOURCE_DERIVATION") + BINARIES_DERIVATION = trust env_var_get("NIX_BINS_DERIVATION") +} + +load_features() +echo SOURCE_CACHE diff --git a/src/platform.ab b/src/platform.ab new file mode 100644 index 0000000..c9dcdd0 --- /dev/null +++ b/src/platform.ab @@ -0,0 +1,46 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Environment-specific data retrieval utilities + +import { env_var_get } from "std/env" + +pub fun get_osname(): Text { + return trust $uname -s$ +} + +pub fun get_machine(): Text { + return trust $uanme -m$ +} + +pub fun get_system(): Text { + let osname = get_osname() + let machine = get_machine() + + return "{osname}.{machine}" +} + +pub fun get_dll_ext(): Text { + let osname = get_osname() + if { + osname == "Darwin": return "dylib" + else: return "so" + } +} + +pub fun get_nix_root(): Text { + let userhome = trust env_var_get("HOME") + let osname = get_osname() + if { + osname == "Darwin": return "{userhome}/Library/Nix" + else: return "{userhome}/.local/share/nix/root" + } +} + +pub fun get_cache_root(): Text { + let userhome = trust env_var_get("HOME") + let osname = get_osname() + if { + osname == "Darwin": return "{userhome}/Library/Caches" + else: return "{userhome}/.cache" + } +} From 59b1fc12d09c8af4fb52d2a4e33bb1b06cc023a7 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Thu, 6 Feb 2025 20:08:59 +0100 Subject: [PATCH 02/22] Improved tarball support --- nixie/output/script.py | 9 ++++++++- src/builder.ab | 3 +++ src/cli.ab | 3 +++ src/common.ab | 44 ++++++++++++++++++++++++++++++++++++++++-- src/main.ab | 26 ++++++++++++++++++++++--- src/resources.ab | 34 ++++++++++++++++++++++++++++++++ 6 files changed, 113 insertions(+), 6 deletions(-) create mode 100644 src/resources.ab diff --git a/nixie/output/script.py b/nixie/output/script.py index 82d6d94..9bfc16c 100644 --- a/nixie/output/script.py +++ b/nixie/output/script.py @@ -63,6 +63,10 @@ class NixieScript: tarball: ResourceTarball = None fname: Path + # Guard value to prevent overrun since Amber scripts lack a default exit + GUARD = b'exit 0\ncat < +// Licensed under GNU GPLv2 +// Functions to build Nix and its dependencies from source diff --git a/src/cli.ab b/src/cli.ab index e69de29..9959239 100644 --- a/src/cli.ab +++ b/src/cli.ab @@ -0,0 +1,3 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Command line options parser diff --git a/src/common.ab b/src/common.ab index 0dcdfc4..c47da25 100644 --- a/src/common.ab +++ b/src/common.ab @@ -6,6 +6,8 @@ import { echo_error } from "std/env" let SELF = trust $echo \$0$ +// Exit alt-buffer, quit script, kill ourselves. At this point something has gone +// catastrophically wrong and the script MUST stop. pub fun bail(message: Text): Null { trust $tput rmcup$ @@ -15,9 +17,47 @@ pub fun bail(message: Text): Null trust $kill -ABRT \$\$$ } -pub fun untar(member: Text): Text +fun dump_archive(): Text { - return "SOURCE_CACHE=xxx" + let dest = trust $mktemp -t nixie_XXXXXXXX.tar$ + + // This block seeks to the archive marker, then zcats everything afterwards + // to the tmpfile in dest + //TODO: port better detection for SELF + $( + read -r M + while ! [[ "\$M" =~ ^-----BEGIN\ ARCHIVE\ SECTION----- ]] + do read -r M + done + gzip -d -c 2>/dev/null > {dest} + ) < {SELF}$ failed { + // gzip exits with 2 IFF it sees trailing garbage, which our alt-buf + // attempt at hiding the tarball technically is + if status != 2 { + bail("Could not find the script's resource archive.") + } + } + + return dest +} + +pub fun untar(member: Text, dump: Bool): Text? +{ + let archive = dump_archive() + let tar_cmd = "tar -x {member} -f {archive}" + + if dump { + tar_cmd = "tar -x -O {member} -f {archive}" + } + + let tar_out = ${tar_cmd}$ failed { + trust $rm {archive}$ + fail 1 + } + + trust $rm {archive}$ + + return tar_out } pub fun check_deps(deps: []): Bool diff --git a/src/main.ab b/src/main.ab index be47660..b38cee1 100644 --- a/src/main.ab +++ b/src/main.ab @@ -10,18 +10,38 @@ let SOURCE_CACHE = "" let SOURCE_DERIVATION = "" let BINARIES_DERIVATION = "" +let EXTRA_FEATURES = "" +let EXTRA_SUBSTITUTERS = "" +let EXTRA_KEYS = "" + +let NIXIE_VERSION = "" + check_deps(["tar", "gzip", "uname"]) fun load_features(): Null { - $eval {untar("features")}$ failed { + let envfile = untar("features", true) failed { bail("The resource archive is missing or malformed.") } + $eval {envfile}$ failed { + bail("The environment file in the resource archive is malformed.") + } + SOURCE_CACHE = trust env_var_get("SOURCE_CACHE") SOURCE_DERIVATION = trust env_var_get("SOURCE_DERIVATION") BINARIES_DERIVATION = trust env_var_get("NIX_BINS_DERIVATION") + + EXTRA_FEATURES = trust env_var_get("EXTRA_FEATURES") + EXTRA_SUBSTITUTERS = trust env_var_get("EXTRA_SUBSTITUTERS") + EXTRA_KEYS = trust env_var_get("EXTRA_TRUSTED_PUBLIC_KEYS") + + NIXIE_VERSION = trust env_var_get("NIXIE_VERSION") } -load_features() -echo SOURCE_CACHE +main(args) +{ + load_features() + echo SOURCE_DERIVATION + echo "Built using Nixie {NIXIE_VERSION}" +} diff --git a/src/resources.ab b/src/resources.ab new file mode 100644 index 0000000..0f00b7a --- /dev/null +++ b/src/resources.ab @@ -0,0 +1,34 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Resource retrieval from tarball or Cachix + +import { file_download } from "std/http" + +import { untar } from "./common.ab" + +fun cachix_url(derivation: Text, member: Text): Text +{ + return "https://{SOURCE_CACHE}/serve/{derivation}/{member}" +} + +pub fun pull_source_file(member: Text, dest: Text): Null? +{ + let where = untar("sources/{member}", false) failed { + let tmpf = trust $mktemp -t nixie_src_XXXXXXXX.tgz$ + let tmpd = trust $mktemp -t -d nixie_{member}_XXXXXXXX$ + file_download(cachix_url(SOURCE_DERIVATION, "{member}.tar.gz"), tmpf)? + trust $rm -f {tmpf}$ + return tmpd + } + trust $mv {where} {dest}$ +} + +pub fun pull_binary(member: Text, dest: Text): Null? +{ + let where = untar(member, false) failed { + let tmpf = trust $mktemp -t nixie_{member}_XXXXXXXX$ + file_download(cachix_url(BINARIES_DERIVATION, member), tmpf)? + return tmpf + } + trust $mv {where} {dest}$ +} From a83be78f72648a59a7b27e068c3b9069a1a36a90 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Thu, 6 Feb 2025 23:01:43 +0100 Subject: [PATCH 03/22] Added --nixie- command line options --- src/cli.ab | 105 +++++++++++++++++++++++++++++++++++++++++++++++ src/common.ab | 41 +++++++++++++----- src/main.ab | 26 ++++++++---- src/nix.ab | 8 ++++ src/resources.ab | 8 ++-- 5 files changed, 165 insertions(+), 23 deletions(-) create mode 100644 src/nix.ab diff --git a/src/cli.ab b/src/cli.ab index 9959239..38433f6 100644 --- a/src/cli.ab +++ b/src/cli.ab @@ -1,3 +1,108 @@ // Nixie © Karim Vergnes // Licensed under GNU GPLv2 // Command line options parser + +import { array_remove_at } from "std/array" +import { starts_with, slice } from "std/text" +import { echo_error, echo_warning, env_var_get, env_var_set } from "std/env" +import { dir_create } from "std/fs" + +import { untar } from "./common.ab" +import { get_nix_root, get_cache_root } from "./platform.ab" + +let NIXIE_VERSION = trust env_var_get("NIXIE_VERSION") + +fun cmd_help() +{ + let NIXIE_VERSION = trust env_var_get("NIXIE_VERSION") + + echo "Nix wrapper script, generated by Nixie {NIXIE_VERSION}" + echo "" + echo "Available --nixie- options:" + + echo " --nixie-help Show this help message and exit." + echo " --nixie-print-config Print this script's configuration." + echo " --nixie-extract Unpack the resources archive into nixie/" + echo " --nixie-cleanup Delete local Nix build files" + echo " --nixie-ignore-system Behave as if Nix was not installed." + + exit 0 +} + +fun cmd_print_config() +{ + echo trust untar("features", true) + exit 0 +} + +fun cmd_extract() +{ + dir_create("nixie") + cd "nixie" + trust untar("") + exit 0 +} + +fun cmd_cleanup() +{ + let nix_root = get_nix_root() + let cache_root = get_cache_root() + + echo "Removing local Nix channels and build files..." + + echo "Removing user Nix store..." + $chmod -R +wx {nix_root}$ failed { + echo_error("Failed to open permissions of the local Nix store.", 0) + echo_error("Try running 'chmod -R +wx {nix_root}' manually.") + } + trust $rm -rf {nix_root}$ + + echo "Removing retrieved Nix binaries..." + trust $rm -rf {cache_root}/nix-static {cache_root}/nix-lib$ + + exit 0 +} + +fun opt_ignore_system() +{ + echo_warning("Ignoring system-wide Nix for testing purposes.") + echo_warning("Re-run without the --nixie-ignore-system flag to import the single-user") + echo_warning("Nix store into the system store.") + trust env_var_set("nosystem", "1") +} + +fun notfound(cmd) +{ + let self = trust env_var_get("0") + + echo_error("No such option: --nixie-{cmd}. Run '{self} --nixie-help' for available options.") +} + +fun eval_cmd(cmd) +{ + if { + cmd == "help": cmd_help() + cmd == "print-config": cmd_print_config() + cmd == "extract": cmd_extract() + cmd == "cleanup": cmd_cleanup() + cmd == "ignore-system": opt_ignore_system() + else: notfound(cmd) + } +} + +pub fun catch_args(ref args: []): Null +{ + let newargs = [Text] + + for arg in args { + let cmd = "" + if starts_with(arg, "--nixie-") { + cmd = slice(arg, len("--nixie-")) + eval_cmd(cmd) + } else { + newargs += [arg] + } + } + + args = newargs +} diff --git a/src/common.ab b/src/common.ab index c47da25..e774d58 100644 --- a/src/common.ab +++ b/src/common.ab @@ -2,9 +2,31 @@ // Licensed under GNU GPLv2 // Common utility functions -import { echo_error } from "std/env" +import { echo_error, env_var_get } from "std/env" +import { starts_with, text_contains } from "std/text" -let SELF = trust $echo \$0$ +let SELF = "" + +let me = trust $echo \$0$ + +// Quick shenanigan to get real path without readlink -f *sideyes macOS* +if text_contains(me, "/") { + if not starts_with(me, "/") { + let PWD = trust env_var_get("PWD") + let rl = $readlink {me}$ failed { + SELF = "{PWD}/{me}" + } + + // This second pass check if the result of readlink is relative + if { + rl == "": SELF = SELF + not starts_with(rl, "/"): SELF = "{PWD}/{rl}" + else: SELF = rl + } + } +} else { + echo_error("This script must be run from an absolute or relative path.") +} // Exit alt-buffer, quit script, kill ourselves. At this point something has gone // catastrophically wrong and the script MUST stop. @@ -12,7 +34,7 @@ pub fun bail(message: Text): Null { trust $tput rmcup$ - echo_error(message, 0) + echo_error(message, 1) echo_error("This script can be rebuilt using the nixie tool.",1) trust $kill -ABRT \$\$$ } @@ -23,14 +45,13 @@ fun dump_archive(): Text // This block seeks to the archive marker, then zcats everything afterwards // to the tmpfile in dest - //TODO: port better detection for SELF - $( + $cat {SELF} | ( read -r M while ! [[ "\$M" =~ ^-----BEGIN\ ARCHIVE\ SECTION----- ]] - do read -r M + do read -r M || return 1 done gzip -d -c 2>/dev/null > {dest} - ) < {SELF}$ failed { + )$ failed { // gzip exits with 2 IFF it sees trailing garbage, which our alt-buf // attempt at hiding the tarball technically is if status != 2 { @@ -41,7 +62,7 @@ fun dump_archive(): Text return dest } -pub fun untar(member: Text, dump: Bool): Text? +pub fun untar(member: Text, dump: Bool = false): Text? { let archive = dump_archive() let tar_cmd = "tar -x {member} -f {archive}" @@ -52,7 +73,7 @@ pub fun untar(member: Text, dump: Bool): Text? let tar_out = ${tar_cmd}$ failed { trust $rm {archive}$ - fail 1 + fail status } trust $rm {archive}$ @@ -70,7 +91,7 @@ pub fun check_deps(deps: []): Bool } } - if "{missing}" != "" { + if len(missing) > 0 { echo_error("The following commands are missing:", 0) for cmd in missing { echo_error("- {cmd}", 0) diff --git a/src/main.ab b/src/main.ab index b38cee1..93a67de 100644 --- a/src/main.ab +++ b/src/main.ab @@ -2,20 +2,24 @@ // Licensed under GNU GPLv2 // Entry point for script logic -import { env_var_get } from "std/env" - -import { bail, untar, check_deps } from "./common.ab" - let SOURCE_CACHE = "" let SOURCE_DERIVATION = "" -let BINARIES_DERIVATION = "" +let NIX_BINS_DERIVATION = "" let EXTRA_FEATURES = "" let EXTRA_SUBSTITUTERS = "" -let EXTRA_KEYS = "" +let EXTRA_TRUSTED_PUBLIC_KEYS = "" let NIXIE_VERSION = "" +let args = [Text] + +import { env_var_get } from "std/env" + +import { bail, untar, check_deps } from "./common.ab" + +import { catch_args } from "./cli.ab" + check_deps(["tar", "gzip", "uname"]) fun load_features(): Null @@ -30,18 +34,22 @@ fun load_features(): Null SOURCE_CACHE = trust env_var_get("SOURCE_CACHE") SOURCE_DERIVATION = trust env_var_get("SOURCE_DERIVATION") - BINARIES_DERIVATION = trust env_var_get("NIX_BINS_DERIVATION") + NIX_BINS_DERIVATION = trust env_var_get("NIX_BINS_DERIVATION") EXTRA_FEATURES = trust env_var_get("EXTRA_FEATURES") EXTRA_SUBSTITUTERS = trust env_var_get("EXTRA_SUBSTITUTERS") - EXTRA_KEYS = trust env_var_get("EXTRA_TRUSTED_PUBLIC_KEYS") + EXTRA_TRUSTED_PUBLIC_KEYS = trust env_var_get("EXTRA_TRUSTED_PUBLIC_KEYS") NIXIE_VERSION = trust env_var_get("NIXIE_VERSION") } -main(args) +main(cmdl) { + // In Amber, cmdl is read-only, but catch_args needs read-write + args = cmdl + load_features() + catch_args(args) echo SOURCE_DERIVATION echo "Built using Nixie {NIXIE_VERSION}" } diff --git a/src/nix.ab b/src/nix.ab new file mode 100644 index 0000000..aaad27e --- /dev/null +++ b/src/nix.ab @@ -0,0 +1,8 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Final-step Nix invocation methods + +fun is_ignore_system(): Bool +{ + return env_var_test("nosystem") +} diff --git a/src/resources.ab b/src/resources.ab index 0f00b7a..f65be25 100644 --- a/src/resources.ab +++ b/src/resources.ab @@ -13,22 +13,22 @@ fun cachix_url(derivation: Text, member: Text): Text pub fun pull_source_file(member: Text, dest: Text): Null? { - let where = untar("sources/{member}", false) failed { + let where = untar("sources/{member}") failed { let tmpf = trust $mktemp -t nixie_src_XXXXXXXX.tgz$ let tmpd = trust $mktemp -t -d nixie_{member}_XXXXXXXX$ file_download(cachix_url(SOURCE_DERIVATION, "{member}.tar.gz"), tmpf)? trust $rm -f {tmpf}$ return tmpd } - trust $mv {where} {dest}$ + mv where dest } pub fun pull_binary(member: Text, dest: Text): Null? { - let where = untar(member, false) failed { + let where = untar(member) failed { let tmpf = trust $mktemp -t nixie_{member}_XXXXXXXX$ file_download(cachix_url(BINARIES_DERIVATION, member), tmpf)? return tmpf } - trust $mv {where} {dest}$ + mv where dest } From 7b9a1fbbeef6348adbbffd01bcfbb78ef0d3253b Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Fri, 7 Feb 2025 14:54:00 +0100 Subject: [PATCH 04/22] Added repo root detection --- src/cli.ab | 10 +++++----- src/common.ab | 5 +++++ src/main.ab | 2 ++ src/platform.ab | 19 ++++++++++++++++++- src/resources.ab | 4 ++++ 5 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/cli.ab b/src/cli.ab index 38433f6..4de888f 100644 --- a/src/cli.ab +++ b/src/cli.ab @@ -8,7 +8,7 @@ import { echo_error, echo_warning, env_var_get, env_var_set } from "std/env" import { dir_create } from "std/fs" import { untar } from "./common.ab" -import { get_nix_root, get_cache_root } from "./platform.ab" +import { get_nix_root, get_cache_root, get_repo_root } from "./platform.ab" let NIXIE_VERSION = trust env_var_get("NIXIE_VERSION") @@ -47,14 +47,14 @@ fun cmd_cleanup() { let nix_root = get_nix_root() let cache_root = get_cache_root() + let repo_root = get_repo_root() echo "Removing local Nix channels and build files..." + trust $chmod -R +wx {repo_root}/.nixie 2>/dev/null$ + trust $rm -rf {repo_root}/.nixie$ echo "Removing user Nix store..." - $chmod -R +wx {nix_root}$ failed { - echo_error("Failed to open permissions of the local Nix store.", 0) - echo_error("Try running 'chmod -R +wx {nix_root}' manually.") - } + trust $chmod -R +wx {nix_root} 2>/dev/null$ trust $rm -rf {nix_root}$ echo "Removing retrieved Nix binaries..." diff --git a/src/common.ab b/src/common.ab index e774d58..c70ba8c 100644 --- a/src/common.ab +++ b/src/common.ab @@ -39,6 +39,11 @@ pub fun bail(message: Text): Null trust $kill -ABRT \$\$$ } +pub fun get_self(): Text +{ + return SELF +} + fun dump_archive(): Text { let dest = trust $mktemp -t nixie_XXXXXXXX.tar$ diff --git a/src/main.ab b/src/main.ab index 93a67de..3c6e261 100644 --- a/src/main.ab +++ b/src/main.ab @@ -18,6 +18,8 @@ import { env_var_get } from "std/env" import { bail, untar, check_deps } from "./common.ab" +import { get_repo_root } from "./platform.ab" + import { catch_args } from "./cli.ab" check_deps(["tar", "gzip", "uname"]) diff --git a/src/platform.ab b/src/platform.ab index c9dcdd0..451c000 100644 --- a/src/platform.ab +++ b/src/platform.ab @@ -2,7 +2,11 @@ // Licensed under GNU GPLv2 // Environment-specific data retrieval utilities -import { env_var_get } from "std/env" +import { env_var_get, echo_warning } from "std/env" +import { split, join } from "std/text" +import { array_pop } from "std/array" + +import { get_self } from "./common.ab" pub fun get_osname(): Text { return trust $uname -s$ @@ -44,3 +48,16 @@ pub fun get_cache_root(): Text { else: return "{userhome}/.cache" } } + +pub fun get_repo_root(): Text +{ + let self_a = split(get_self(), "/") + array_pop(self_a) + + let self_dir = "/{join(self_a, "/")}" + + return $git -C {self_dir} rev-parse --show-toplevel$ failed { + echo_warning("Failed to find current Git repository, using script parent directory.") + return self_dir + } +} diff --git a/src/resources.ab b/src/resources.ab index f65be25..e2f76a1 100644 --- a/src/resources.ab +++ b/src/resources.ab @@ -16,10 +16,12 @@ pub fun pull_source_file(member: Text, dest: Text): Null? let where = untar("sources/{member}") failed { let tmpf = trust $mktemp -t nixie_src_XXXXXXXX.tgz$ let tmpd = trust $mktemp -t -d nixie_{member}_XXXXXXXX$ + file_download(cachix_url(SOURCE_DERIVATION, "{member}.tar.gz"), tmpf)? trust $rm -f {tmpf}$ return tmpd } + mv where dest } @@ -27,8 +29,10 @@ pub fun pull_binary(member: Text, dest: Text): Null? { let where = untar(member) failed { let tmpf = trust $mktemp -t nixie_{member}_XXXXXXXX$ + file_download(cachix_url(BINARIES_DERIVATION, member), tmpf)? return tmpf } + mv where dest } From 267e3d42bd7dd4299dc8e7f7a2fb484163c320f9 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Sat, 8 Feb 2025 14:54:03 +0100 Subject: [PATCH 05/22] Implementing builder, wrapped amber in faketime --- default.nix | 22 ++++++++++++++++---- src/builder.ab | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/common.ab | 10 +++++++--- src/main.ab | 3 +-- src/nix.ab | 17 ++++++++++++++++ src/term.ab | 23 +++++++++++++++++++++ 6 files changed, 120 insertions(+), 9 deletions(-) create mode 100644 src/term.ab diff --git a/default.nix b/default.nix index 9660a17..ba52691 100644 --- a/default.nix +++ b/default.nix @@ -1,14 +1,15 @@ -{ lib, python3Packages, fetchPypi, nix-index, nix, hatch, amber-lang, ... }: +{ lib, python3Packages, fetchPypi, nix-index, nix, hatch, amber-lang +, libfaketime, ... }: let nixie_ver = "2023.09-a2"; pzp = python3Packages.buildPythonPackage rec { pname = "pzp"; - version = "0.0.22"; + version = "0.0.27"; src = fetchPypi { inherit pname version; - sha256 = "sha256-RPx0nnB9+cC/n7eOX0hF21TxM/yEkGy/akRnqV/YN8E="; + sha256 = "sha256-HwsOJA3spcCUFvIBLFkSpMIPZgcU9r1JkAgt0Cj/oUg="; }; doCheck = false; @@ -28,6 +29,18 @@ let sha256 = "sha256-x4UgmGkH5HU48suyT95B7fwsxV9FK1Ni+64Vzk5jRPc="; }; }; + + # By default, Amber adds a header with the build time, which can't be disabled. + # This wraps the compiler in faketime to make the timestamp reproducible. + # See amber-lang/amber#672 + amber-reproducible = amber-lang.overrideAttrs (self: super: { + postInstall = '' + ${super.postInstall} + mv $out/bin/amber $out/bin/.amber-orig + makeWrapper ${libfaketime}/bin/faketime $out/bin/amber \ + --add-flags "-f '1970-01-01 01:00:00' $out/bin/.amber-orig" + ''; + }); in python3Packages.buildPythonApplication { pname = "nixie"; version = nixie_ver; @@ -38,7 +51,8 @@ in python3Packages.buildPythonApplication { nativeBuildInputs = [ hatch hatch-build-scripts - amber-lang + amber-reproducible + libfaketime ]; propagatedBuildInputs = with python3Packages; diff --git a/src/builder.ab b/src/builder.ab index 312061d..54587e0 100644 --- a/src/builder.ab +++ b/src/builder.ab @@ -1,3 +1,57 @@ // Nixie © Karim Vergnes // Licensed under GNU GPLv2 // Functions to build Nix and its dependencies from source + +import { check_deps } from "./common.ab" +import { pull_source_file, pull_binary } from "./resources.ab" + +fun build_openssl() +{ + +} + + +fun build_lowdown() +{ + +} + +fun build_nlohmann_json() +{ + +} + +fun build_boost() +{ + +} + +/// Build a dependency which uses Autoconf as its build system. +/// +/// ### Arguments: +/// - `lib_name`: The library to check for with `pkg-config` +/// - `var_name`: The variable to export library locations +/// - `lib_prefix`: Where in the source tree the resulting libraries are found +/// - `inc_prefix`: Where in the source tree the build headers are found +fun build_autoconf_dep(lib_name: Text, var_name: Text, + lib_prefix: Text = "", + inc_prefix: Text = ""): Null +{ + +} + +/// Export libraries found inside the macOS SDK for building Nix +fun darwin_export_sdk() +{ + +} + +/// Build Nix and its dependencies locally, then place it in the expected location +/// in the user's cache directory. +/// +/// This process **requires**, among other things, `pkg-config` to determine +/// which dependencies are already met and avoid building them. +pub fun build_nix() +{ + check_deps(["cc", "c++", "pkg-config", "make", "flex", "bison", "perl"]) +} diff --git a/src/common.ab b/src/common.ab index c70ba8c..7479463 100644 --- a/src/common.ab +++ b/src/common.ab @@ -2,9 +2,11 @@ // Licensed under GNU GPLv2 // Common utility functions -import { echo_error, env_var_get } from "std/env" +import { echo_error, env_var_get, is_command } from "std/env" import { starts_with, text_contains } from "std/text" +import { exit_alt_buffer, set_title } from "./term.ab" + let SELF = "" let me = trust $echo \$0$ @@ -32,10 +34,12 @@ if text_contains(me, "/") { // catastrophically wrong and the script MUST stop. pub fun bail(message: Text): Null { - trust $tput rmcup$ + exit_alt_buffer() + set_title() // Clear terminal title back to default echo_error(message, 1) echo_error("This script can be rebuilt using the nixie tool.",1) + trust $kill -ABRT \$\$$ } @@ -91,7 +95,7 @@ pub fun check_deps(deps: []): Bool let missing = [ Text ] for dep in deps { - silent $which {dep}$ failed { + if not is_command(dep) { missing += [ dep ] } } diff --git a/src/main.ab b/src/main.ab index 3c6e261..cdaa12e 100644 --- a/src/main.ab +++ b/src/main.ab @@ -17,10 +17,9 @@ let args = [Text] import { env_var_get } from "std/env" import { bail, untar, check_deps } from "./common.ab" - import { get_repo_root } from "./platform.ab" - import { catch_args } from "./cli.ab" +import { launch_nix } from "./nix.ab" check_deps(["tar", "gzip", "uname"]) diff --git a/src/nix.ab b/src/nix.ab index aaad27e..bb531c1 100644 --- a/src/nix.ab +++ b/src/nix.ab @@ -2,7 +2,24 @@ // Licensed under GNU GPLv2 // Final-step Nix invocation methods +import { build_nix } from "./builder.ab" + fun is_ignore_system(): Bool { return env_var_test("nosystem") } + +fun get_nix() +{ + +} + +fun launch_darwin_workaround() +{ + +} + +pub fun launch_nix(name: Text, extras: [Text], args: [Text]): Null +{ + +} diff --git a/src/term.ab b/src/term.ab new file mode 100644 index 0000000..a2f33ce --- /dev/null +++ b/src/term.ab @@ -0,0 +1,23 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Utilities to manipulate Xterm features + +let tsl = trust $tput tsl$ //TODO: improve logic to match xterm+sl +let fsl = trust $tput fsl$ //TODO: as above +let smcup = trust $tput smcup$ +let rmcup = trust $tput rmcup$ + +pub fun set_title(title: Text = ""): Null +{ + echo "{tsl}{title}{fsl}" +} + +pub fun enter_alt_buffer(): Null +{ + echo smcup +} + +pub fun exit_alt_buffer(): Null +{ + echo rmcup +} From 4e223b888d1b7875f4114a5543c48e2dd41512f0 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Sat, 8 Feb 2025 22:51:17 +0100 Subject: [PATCH 06/22] Made check_deps failable --- src/cli.ab | 12 +++++------- src/common.ab | 14 ++++++-------- src/main.ab | 34 ++++++++++++++++++---------------- src/nix.ab | 9 +++++++++ 4 files changed, 38 insertions(+), 31 deletions(-) diff --git a/src/cli.ab b/src/cli.ab index 4de888f..3269bce 100644 --- a/src/cli.ab +++ b/src/cli.ab @@ -2,15 +2,13 @@ // Licensed under GNU GPLv2 // Command line options parser -import { array_remove_at } from "std/array" -import { starts_with, slice } from "std/text" +import { array_remove_at } from "std/array" +import { starts_with, slice } from "std/text" import { echo_error, echo_warning, env_var_get, env_var_set } from "std/env" -import { dir_create } from "std/fs" +import { dir_create } from "std/fs" -import { untar } from "./common.ab" -import { get_nix_root, get_cache_root, get_repo_root } from "./platform.ab" - -let NIXIE_VERSION = trust env_var_get("NIXIE_VERSION") +import { untar } from "./common.ab" +import { get_nix_root, get_cache_root, get_repo_root } from "./platform.ab" fun cmd_help() { diff --git a/src/common.ab b/src/common.ab index 7479463..f7e3a5d 100644 --- a/src/common.ab +++ b/src/common.ab @@ -2,14 +2,14 @@ // Licensed under GNU GPLv2 // Common utility functions -import { echo_error, env_var_get, is_command } from "std/env" -import { starts_with, text_contains } from "std/text" +import { echo_error, env_var_get, is_command } from "std/env" +import { starts_with, text_contains } from "std/text" -import { exit_alt_buffer, set_title } from "./term.ab" +import { exit_alt_buffer, set_title } from "./term.ab" let SELF = "" -let me = trust $echo \$0$ +let me = trust env_var_get("0") // Quick shenanigan to get real path without readlink -f *sideyes macOS* if text_contains(me, "/") { @@ -90,7 +90,7 @@ pub fun untar(member: Text, dump: Bool = false): Text? return tar_out } -pub fun check_deps(deps: []): Bool +pub fun check_deps(deps: []): Null? { let missing = [ Text ] @@ -106,8 +106,6 @@ pub fun check_deps(deps: []): Bool echo_error("- {cmd}", 0) } echo_error("Use your distribution's package manager to install them, then try again.", 0) - return false + fail 1 } - - return true } diff --git a/src/main.ab b/src/main.ab index cdaa12e..eb22003 100644 --- a/src/main.ab +++ b/src/main.ab @@ -2,26 +2,28 @@ // Licensed under GNU GPLv2 // Entry point for script logic -let SOURCE_CACHE = "" -let SOURCE_DERIVATION = "" +let SOURCE_CACHE = "" +let SOURCE_DERIVATION = "" let NIX_BINS_DERIVATION = "" -let EXTRA_FEATURES = "" -let EXTRA_SUBSTITUTERS = "" -let EXTRA_TRUSTED_PUBLIC_KEYS = "" +let EXTRA_FEATURES = "" +let EXTRA_SUBSTITUTERS = "" +let EXTRA_TRUSTED_PUBLIC_KEYS = "" let NIXIE_VERSION = "" let args = [Text] -import { env_var_get } from "std/env" +import { env_var_get } from "std/env" -import { bail, untar, check_deps } from "./common.ab" -import { get_repo_root } from "./platform.ab" -import { catch_args } from "./cli.ab" -import { launch_nix } from "./nix.ab" +import { bail, untar, check_deps } from "./common.ab" +import { get_repo_root } from "./platform.ab" +import { catch_args } from "./cli.ab" +import { launch_nix } from "./nix.ab" -check_deps(["tar", "gzip", "uname"]) +check_deps(["tar", "gzip", "uname"]) failed { + exit 1 +} fun load_features(): Null { @@ -33,13 +35,13 @@ fun load_features(): Null bail("The environment file in the resource archive is malformed.") } - SOURCE_CACHE = trust env_var_get("SOURCE_CACHE") - SOURCE_DERIVATION = trust env_var_get("SOURCE_DERIVATION") + SOURCE_CACHE = trust env_var_get("SOURCE_CACHE") + SOURCE_DERIVATION = trust env_var_get("SOURCE_DERIVATION") NIX_BINS_DERIVATION = trust env_var_get("NIX_BINS_DERIVATION") - EXTRA_FEATURES = trust env_var_get("EXTRA_FEATURES") - EXTRA_SUBSTITUTERS = trust env_var_get("EXTRA_SUBSTITUTERS") - EXTRA_TRUSTED_PUBLIC_KEYS = trust env_var_get("EXTRA_TRUSTED_PUBLIC_KEYS") + EXTRA_FEATURES = trust env_var_get("EXTRA_FEATURES") + EXTRA_SUBSTITUTERS = trust env_var_get("EXTRA_SUBSTITUTERS") + EXTRA_TRUSTED_PUBLIC_KEYS = trust env_var_get("EXTRA_TRUSTED_PUBLIC_KEYS") NIXIE_VERSION = trust env_var_get("NIXIE_VERSION") } diff --git a/src/nix.ab b/src/nix.ab index bb531c1..3015ac0 100644 --- a/src/nix.ab +++ b/src/nix.ab @@ -19,6 +19,15 @@ fun launch_darwin_workaround() } +/// Launch Nix with options pulled from a file's contents. +/// This allows our script to be used as an interpreter, the same way +/// Nix proper would. +pub fun launch_nix_shebang(name: Text, file: Text, args: [Text]): Null +{ + +} + +/// Launch Nix as a regular CLI. pub fun launch_nix(name: Text, extras: [Text], args: [Text]): Null { From 005c5c5fd05dce04d385bb92c3e3423d4ee55a6f Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Sun, 9 Feb 2025 00:50:30 +0100 Subject: [PATCH 07/22] Implementing alt-buffer and build logic --- .gitignore | 2 ++ shell.nix | 2 +- src/builder.ab | 4 +++- src/cli.ab | 2 ++ src/common.ab | 35 +++++++++++++++++++++++++++++------ src/main.ab | 14 ++++++++++---- src/nix.ab | 26 ++++++++++++++++++++++++-- src/term.ab | 9 ++++++++- 8 files changed, 79 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 0e493b3..c4f26d9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ features work __pycache__ result +nixie/output/nix-wrapped.sh.in +src/docs diff --git a/shell.nix b/shell.nix index 6d90865..9857a5f 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ { pkgs ? import {} }: let - builder = pkgs.callPackage ./builder {}; + builder = pkgs.callPackage ./. {}; in pkgs.mkShell { name = "nixie"; diff --git a/src/builder.ab b/src/builder.ab index 54587e0..a7c7d5f 100644 --- a/src/builder.ab +++ b/src/builder.ab @@ -2,8 +2,9 @@ // Licensed under GNU GPLv2 // Functions to build Nix and its dependencies from source -import { check_deps } from "./common.ab" +import { check_deps, bail } from "./common.ab" import { pull_source_file, pull_binary } from "./resources.ab" +import { set_title } from "./term.ab" fun build_openssl() { @@ -54,4 +55,5 @@ fun darwin_export_sdk() pub fun build_nix() { check_deps(["cc", "c++", "pkg-config", "make", "flex", "bison", "perl"]) + failed { bail("Failed to build Nix from source") } } diff --git a/src/cli.ab b/src/cli.ab index 3269bce..0d44193 100644 --- a/src/cli.ab +++ b/src/cli.ab @@ -88,6 +88,8 @@ fun eval_cmd(cmd) } } +/// Parse and remove options in the form `--nixie-[option]` from the arguments +/// list. pub fun catch_args(ref args: []): Null { let newargs = [Text] diff --git a/src/common.ab b/src/common.ab index f7e3a5d..0e747cb 100644 --- a/src/common.ab +++ b/src/common.ab @@ -30,24 +30,35 @@ if text_contains(me, "/") { echo_error("This script must be run from an absolute or relative path.") } -// Exit alt-buffer, quit script, kill ourselves. At this point something has gone -// catastrophically wrong and the script MUST stop. -pub fun bail(message: Text): Null +/// Exit alt-buffer, quit script, kill ourselves. At this point something has gone +/// catastrophically wrong and the script MUST stop. +/// +/// ### Arguments: +/// - `message`: Message to print as we error out +/// - `archive`: `true` if the issue is related to the attached archive. +/// Default is `false`. +pub fun bail(message: Text, archive: Bool = false): Null { exit_alt_buffer() set_title() // Clear terminal title back to default - echo_error(message, 1) - echo_error("This script can be rebuilt using the nixie tool.",1) + echo_error(message, 0) + if archive { + echo_error("This script can be rebuilt using the nixie tool.", 0) + } + exit 1 trust $kill -ABRT \$\$$ } +/// Retrieve the resolved, absolute path to the wrapper script. pub fun get_self(): Text { return SELF } +/// Reads and uncompresses to a temporary file, the attached resource archive +/// at the end of the script. fun dump_archive(): Text { let dest = trust $mktemp -t nixie_XXXXXXXX.tar$ @@ -64,13 +75,21 @@ fun dump_archive(): Text // gzip exits with 2 IFF it sees trailing garbage, which our alt-buf // attempt at hiding the tarball technically is if status != 2 { - bail("Could not find the script's resource archive.") + bail("Could not find the script's resource archive.", true) } } return dest } +/// Extract a file or directory from the attached resource archive at the end +/// of the script. +/// +/// ### Arguments +/// - `member`: The plaintext path to the file or folder to be extracted, relative +/// to the archive root. Passed as-is to `tar`. +/// - `dump`: If `true`, output the contents of the extracted file to standard +/// output. Adds the `-O` option to `tar`. pub fun untar(member: Text, dump: Bool = false): Text? { let archive = dump_archive() @@ -90,6 +109,10 @@ pub fun untar(member: Text, dump: Bool = false): Text? return tar_out } +/// Checks that a list of commands are available at runtime. +/// +/// This function prints out the list of missing commands, and a message to +/// instruct the user to install them. pub fun check_deps(deps: []): Null? { let missing = [ Text ] diff --git a/src/main.ab b/src/main.ab index eb22003..1063b4d 100644 --- a/src/main.ab +++ b/src/main.ab @@ -25,14 +25,20 @@ check_deps(["tar", "gzip", "uname"]) failed { exit 1 } +/// Populate environment variables from the features file in the archive. +/// Also sets variables local to this module, which is not much use until +/// [amber-lang/amber#671](https://github.com/amber-lang/amber/issues/671) is +/// implemented. Checks that the variables exist, I guess. +/// +/// Could be moved to `common.ab` fun load_features(): Null { let envfile = untar("features", true) failed { - bail("The resource archive is missing or malformed.") + bail("The resource archive is missing or malformed.", true) } $eval {envfile}$ failed { - bail("The environment file in the resource archive is malformed.") + bail("The environment file in the resource archive is malformed.", true) } SOURCE_CACHE = trust env_var_get("SOURCE_CACHE") @@ -53,6 +59,6 @@ main(cmdl) load_features() catch_args(args) - echo SOURCE_DERIVATION - echo "Built using Nixie {NIXIE_VERSION}" + + launch_nix("nix", ["lol"], ["lol"]) } diff --git a/src/nix.ab b/src/nix.ab index 3015ac0..c95a120 100644 --- a/src/nix.ab +++ b/src/nix.ab @@ -3,6 +3,8 @@ // Final-step Nix invocation methods import { build_nix } from "./builder.ab" +import { pull_binary } from "./resources.ab" +import { enter_alt_buffer, set_title, teardown } from "./term.ab" fun is_ignore_system(): Bool { @@ -11,7 +13,20 @@ fun is_ignore_system(): Bool fun get_nix() { + enter_alt_buffer() + set_title("Building Nix...") + // Cleanly exit alt-buffer if hit with ^C + // Right now amber-lang/amber#673 makes this impossible + //trust $trap "{nameof teardown}; exit 1" SIGKILL SIGTERM SIGINT SIGABRT$ + + //TODO: ✨MAGIC✨ + echo "Pretend we're doing something here..." + trust $sleep 1$ + + // By this point every attempt at fetching Nix failed. + teardown() + fail 1 } fun launch_darwin_workaround() @@ -19,7 +34,7 @@ fun launch_darwin_workaround() } -/// Launch Nix with options pulled from a file's contents. +/// Launch Nix with options pulled from a file's contents. /// This allows our script to be used as an interpreter, the same way /// Nix proper would. pub fun launch_nix_shebang(name: Text, file: Text, args: [Text]): Null @@ -30,5 +45,12 @@ pub fun launch_nix_shebang(name: Text, file: Text, args: [Text]): Null /// Launch Nix as a regular CLI. pub fun launch_nix(name: Text, extras: [Text], args: [Text]): Null { - + trust get_nix() + + echo "Would launch Nix as {name}" + echo "Internal options: {extras}" + echo "Command line options: {args}" + echo "" + echo "That's all we have for now, thanks for playing!" + exit 0 } diff --git a/src/term.ab b/src/term.ab index a2f33ce..ba17898 100644 --- a/src/term.ab +++ b/src/term.ab @@ -9,7 +9,7 @@ let rmcup = trust $tput rmcup$ pub fun set_title(title: Text = ""): Null { - echo "{tsl}{title}{fsl}" + echo "{tsl}{title} {fsl}" } pub fun enter_alt_buffer(): Null @@ -21,3 +21,10 @@ pub fun exit_alt_buffer(): Null { echo rmcup } + +/// Shorthand to exit alt-buffer and clear terminal title. +pub fun teardown(): Null +{ + set_title() + exit_alt_buffer() +} From 54a71160dbd2eb522378ed1ff01648bc693c3e23 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Sun, 9 Feb 2025 10:10:07 +0100 Subject: [PATCH 08/22] Moving to my fork for nameof functionality --- flake.lock | 11 ++++++----- flake.nix | 2 +- src/nix.ab | 3 +-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/flake.lock b/flake.lock index 653c140..5a4e26c 100644 --- a/flake.lock +++ b/flake.lock @@ -8,15 +8,16 @@ "utils": "utils" }, "locked": { - "lastModified": 1734811072, - "narHash": "sha256-N9G/2G8+vrpr1/K7XLwgW+X2oAyAaz4qvN+EbLOCU1Q=", - "owner": "amber-lang", + "lastModified": 1739091767, + "narHash": "sha256-6z1+TMi9lUdIWGz/RDyhCR69lCrl/tyJ0Ezfp2c18aI=", + "owner": "thesola10", "repo": "amber", - "rev": "d3ceda317fc053ae9645e9f3c8f166657eba24e9", + "rev": "7ba5ae7b20f0b08b504c2b9e8b96b0094ef39485", "type": "github" }, "original": { - "owner": "amber-lang", + "owner": "thesola10", + "ref": "nameof-function", "repo": "amber", "type": "github" } diff --git a/flake.nix b/flake.nix index 26b2d7e..6e8afd7 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "Put Nix in everything!"; inputs.nixpkgs.url = github:nixos/nixpkgs; - inputs.amber.url = github:amber-lang/amber; + inputs.amber.url = github:thesola10/amber/nameof-function; inputs.nix.url = github:nixos/nix/2.17.1; inputs.fakedir = { url = github:nixie-dev/fakedir; diff --git a/src/nix.ab b/src/nix.ab index c95a120..827ebb9 100644 --- a/src/nix.ab +++ b/src/nix.ab @@ -17,8 +17,7 @@ fun get_nix() set_title("Building Nix...") // Cleanly exit alt-buffer if hit with ^C - // Right now amber-lang/amber#673 makes this impossible - //trust $trap "{nameof teardown}; exit 1" SIGKILL SIGTERM SIGINT SIGABRT$ + trust $trap "{nameof teardown}; exit 1" SIGKILL SIGTERM SIGINT SIGABRT$ //TODO: ✨MAGIC✨ echo "Pretend we're doing something here..." From 09f8dd044ed1e2fc45a440bf21ddd7f02374c43e Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Sun, 9 Feb 2025 15:14:15 +0100 Subject: [PATCH 09/22] Implementing Nix binary retrieval --- src/builder.ab | 2 +- src/common.ab | 3 ++- src/main.ab | 1 - src/nix.ab | 58 ++++++++++++++++++++++++++++++++++++++++++------ src/platform.ab | 38 ++++++++++++++++++++++--------- src/resources.ab | 31 +++++++++++++++++++------- 6 files changed, 105 insertions(+), 28 deletions(-) diff --git a/src/builder.ab b/src/builder.ab index a7c7d5f..fa713ff 100644 --- a/src/builder.ab +++ b/src/builder.ab @@ -2,9 +2,9 @@ // Licensed under GNU GPLv2 // Functions to build Nix and its dependencies from source +import { set_title } from "./term.ab" import { check_deps, bail } from "./common.ab" import { pull_source_file, pull_binary } from "./resources.ab" -import { set_title } from "./term.ab" fun build_openssl() { diff --git a/src/common.ab b/src/common.ab index 0e747cb..8ee55b4 100644 --- a/src/common.ab +++ b/src/common.ab @@ -100,8 +100,9 @@ pub fun untar(member: Text, dump: Bool = false): Text? } let tar_out = ${tar_cmd}$ failed { + let tar_status = status trust $rm {archive}$ - fail status + fail tar_status } trust $rm {archive}$ diff --git a/src/main.ab b/src/main.ab index 1063b4d..c99d896 100644 --- a/src/main.ab +++ b/src/main.ab @@ -17,7 +17,6 @@ let args = [Text] import { env_var_get } from "std/env" import { bail, untar, check_deps } from "./common.ab" -import { get_repo_root } from "./platform.ab" import { catch_args } from "./cli.ab" import { launch_nix } from "./nix.ab" diff --git a/src/nix.ab b/src/nix.ab index 827ebb9..0b7056f 100644 --- a/src/nix.ab +++ b/src/nix.ab @@ -2,24 +2,63 @@ // Licensed under GNU GPLv2 // Final-step Nix invocation methods -import { build_nix } from "./builder.ab" -import { pull_binary } from "./resources.ab" -import { enter_alt_buffer, set_title, teardown } from "./term.ab" +import { file_exists, dir_exists, dir_create } from "std/fs" -fun is_ignore_system(): Bool +import { enter_alt_buffer, set_title, teardown } from "./term.ab" +import { bail } from "./common.ab" +import { get_osname, get_system, get_cache_root } from "./platform.ab" +import { pull_binary } from "./resources.ab" +import { build_nix } from "./builder.ab" + +/// Determine whether Nix is installed system-wide. Nixie can launch the +/// system-wide Nix when it is available, using the script's bundled options. +/// +/// This is the predicate for that behavior. +fun is_nix_installed(): Bool { - return env_var_test("nosystem") + // --nixie-ignore-system flag set + if env_var_test("nosystem"): return false + + // Simple enough heuristic. In the future, might also check for the `nix` + // command, but that would create a ternary situation where Nixie's static + // Nix would need to use the system store. + if dir_exists("/nix/store"): return true + + return false } +/// Attempt to retrieve the static Nix binary by all known methods. +/// +/// This function will first try to extract or download a static executable +/// for the current system, and if it fails, a local build from source will be +/// attempted. +/// +/// **NOTE**: On macOS, this function is also responsible for downloading +/// `libfakedir.dylib`. fun get_nix() { + let cache_root = get_cache_root() + let osname = get_osname() + let system = get_system() + + let nix_path = "{cache_root}/nix-static" + let fakedir_path = "{cache_root}/nix-lib/libfakedir.dylib" + enter_alt_buffer() set_title("Building Nix...") // Cleanly exit alt-buffer if hit with ^C trust $trap "{nameof teardown}; exit 1" SIGKILL SIGTERM SIGINT SIGABRT$ - //TODO: ✨MAGIC✨ + if osname == "Darwin" and not file_exists(fakedir_path) { + dir_create("{cache_root}/nix-lib") + trust pull_binary("libfakedir.dylib", fakedir_path) + } + + if not file_exists(nix_path) { + trust pull_binary("nix.{system}", nix_path) + } + echo "Pretend we're doing something here..." trust $sleep 1$ @@ -28,7 +67,12 @@ fun get_nix() fail 1 } -fun launch_darwin_workaround() +/// Launch Nix with the proper environment variables for fakedir and OpenSSL. +/// +/// This is required because macOS has no namespacing facility, and Nix will +/// not attempt to use the system-wide CA bundle by default. This function +/// addresses both issues. +fun launch_darwin_workaround(name: Text, args: [Text]): Null { } diff --git a/src/platform.ab b/src/platform.ab index 451c000..832d936 100644 --- a/src/platform.ab +++ b/src/platform.ab @@ -3,27 +3,37 @@ // Environment-specific data retrieval utilities import { env_var_get, echo_warning } from "std/env" -import { split, join } from "std/text" -import { array_pop } from "std/array" +import { split, join } from "std/text" +import { array_pop } from "std/array" import { get_self } from "./common.ab" -pub fun get_osname(): Text { +/// Return the current operating system, as reported by `uname -s`. +pub fun get_osname(): Text +{ return trust $uname -s$ } -pub fun get_machine(): Text { - return trust $uanme -m$ +/// Return the current machine architecture, as reported by `uname -s`. +pub fun get_machine(): Text +{ + return trust $uname -m$ } -pub fun get_system(): Text { +/// Return the combined system-string, in the format `osname.machine`. +/// +/// See [`get_osname()`](#get_osname) and [`get_machine()`](#get_machine) above. +pub fun get_system(): Text +{ let osname = get_osname() let machine = get_machine() return "{osname}.{machine}" } -pub fun get_dll_ext(): Text { +/// Return the extension for dynamic libraries on the current system. +pub fun get_dll_ext(): Text +{ let osname = get_osname() if { osname == "Darwin": return "dylib" @@ -31,7 +41,9 @@ pub fun get_dll_ext(): Text { } } -pub fun get_nix_root(): Text { +/// Return the path to the user-mode root for the Nix store. +pub fun get_nix_root(): Text +{ let userhome = trust env_var_get("HOME") let osname = get_osname() if { @@ -40,7 +52,9 @@ pub fun get_nix_root(): Text { } } -pub fun get_cache_root(): Text { +/// Return the path to the user's cache directory on the current system. +pub fun get_cache_root(): Text +{ let userhome = trust env_var_get("HOME") let osname = get_osname() if { @@ -49,12 +63,16 @@ pub fun get_cache_root(): Text { } } +/// Return the path to the root of the Git repository the script is located in. +/// +/// Falls back to returning the script's parent directory if not in a repository, +/// or if `git` is unavailable. pub fun get_repo_root(): Text { let self_a = split(get_self(), "/") array_pop(self_a) - let self_dir = "/{join(self_a, "/")}" + let self_dir = "/" + join(self_a, "/") return $git -C {self_dir} rev-parse --show-toplevel$ failed { echo_warning("Failed to find current Git repository, using script parent directory.") diff --git a/src/resources.ab b/src/resources.ab index e2f76a1..22927e3 100644 --- a/src/resources.ab +++ b/src/resources.ab @@ -3,36 +3,51 @@ // Resource retrieval from tarball or Cachix import { file_download } from "std/http" +import { env_var_get } from "std/env" import { untar } from "./common.ab" fun cachix_url(derivation: Text, member: Text): Text { + let SOURCE_CACHE = trust env_var_get("SOURCE_CACHE") + return "https://{SOURCE_CACHE}/serve/{derivation}/{member}" } pub fun pull_source_file(member: Text, dest: Text): Null? { - let where = untar("sources/{member}") failed { + let SOURCE_DERIVATION = trust env_var_get("SOURCE_DERIVATION") + + let where = trust untar("sources/{member}") + if status != 0 { let tmpf = trust $mktemp -t nixie_src_XXXXXXXX.tgz$ let tmpd = trust $mktemp -t -d nixie_{member}_XXXXXXXX$ - file_download(cachix_url(SOURCE_DERIVATION, "{member}.tar.gz"), tmpf)? + if not file_download(cachix_url(SOURCE_DERIVATION, "{member}.tar.gz"), tmpf) { + fail 1 + } + trust $rm -f {tmpf}$ - return tmpd + where = tmpd } - mv where dest + trust mv where dest } pub fun pull_binary(member: Text, dest: Text): Null? { - let where = untar(member) failed { + let NIX_BINS_DERIVATION = trust env_var_get("NIX_BINS_DERIVATION") + + let where = trust untar(member) + if status != 0 { let tmpf = trust $mktemp -t nixie_{member}_XXXXXXXX$ - file_download(cachix_url(BINARIES_DERIVATION, member), tmpf)? - return tmpf + if not file_download(cachix_url(NIX_BINS_DERIVATION, member), tmpf) { + fail 1 + } + + where = tmpf } - mv where dest + trust mv where dest } From 9e8e931497fc2b542fe57205b14e92873119bbcc Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Sun, 9 Feb 2025 15:43:29 +0100 Subject: [PATCH 10/22] Finalized get_nix high-level logic --- .gitignore | 1 + src/nix.ab | 25 ++++++++++++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index c4f26d9..d591798 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ __pycache__ result nixie/output/nix-wrapped.sh.in src/docs +src/*.sh diff --git a/src/nix.ab b/src/nix.ab index 0b7056f..19533e6 100644 --- a/src/nix.ab +++ b/src/nix.ab @@ -3,6 +3,7 @@ // Final-step Nix invocation methods import { file_exists, dir_exists, dir_create } from "std/fs" +import { env_var_test } from "std/env" import { enter_alt_buffer, set_title, teardown } from "./term.ab" import { bail } from "./common.ab" @@ -55,16 +56,26 @@ fun get_nix() trust pull_binary("libfakedir.dylib", fakedir_path) } - if not file_exists(nix_path) { - trust pull_binary("nix.{system}", nix_path) + if is_nix_installed() { + teardown() + return 0 } - echo "Pretend we're doing something here..." - trust $sleep 1$ + // We already have static Nix + if file_exists(nix_path) { + teardown() + return 0 + } + + // Try to extract or download Nix + pull_binary("nix.{system}", nix_path) failed { + trust build_nix() + teardown() + fail status + } - // By this point every attempt at fetching Nix failed. teardown() - fail 1 + return 0 } /// Launch Nix with the proper environment variables for fakedir and OpenSSL. @@ -88,7 +99,7 @@ pub fun launch_nix_shebang(name: Text, file: Text, args: [Text]): Null /// Launch Nix as a regular CLI. pub fun launch_nix(name: Text, extras: [Text], args: [Text]): Null { - trust get_nix() + get_nix() failed { echo "oh nyo" } echo "Would launch Nix as {name}" echo "Internal options: {extras}" From d6894f903f1e5a17e60ecbda2c2c53b6e16e89a3 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Sun, 9 Feb 2025 16:19:46 +0100 Subject: [PATCH 11/22] Fixed catch_args, preparing Nix wrapper --- src/cli.ab | 10 +++++----- src/main.ab | 5 ++++- src/nix.ab | 46 +++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/src/cli.ab b/src/cli.ab index 0d44193..509006a 100644 --- a/src/cli.ab +++ b/src/cli.ab @@ -92,17 +92,17 @@ fun eval_cmd(cmd) /// list. pub fun catch_args(ref args: []): Null { - let newargs = [Text] + let local_args = args - for arg in args { + args = [Text] + + for arg in local_args { let cmd = "" if starts_with(arg, "--nixie-") { cmd = slice(arg, len("--nixie-")) eval_cmd(cmd) } else { - newargs += [arg] + args += [arg] } } - - args = newargs } diff --git a/src/main.ab b/src/main.ab index c99d896..357a9fa 100644 --- a/src/main.ab +++ b/src/main.ab @@ -15,6 +15,7 @@ let NIXIE_VERSION = "" let args = [Text] import { env_var_get } from "std/env" +import { array_shift } from "std/array" import { bail, untar, check_deps } from "./common.ab" import { catch_args } from "./cli.ab" @@ -59,5 +60,7 @@ main(cmdl) load_features() catch_args(args) - launch_nix("nix", ["lol"], ["lol"]) + let self = array_shift(args) + + launch_nix(self, ["lol"], args) } diff --git a/src/nix.ab b/src/nix.ab index 19533e6..e966be7 100644 --- a/src/nix.ab +++ b/src/nix.ab @@ -2,12 +2,12 @@ // Licensed under GNU GPLv2 // Final-step Nix invocation methods -import { file_exists, dir_exists, dir_create } from "std/fs" -import { env_var_test } from "std/env" +import { file_exists, dir_exists, dir_create } from "std/fs" +import { env_var_test, env_var_set } from "std/env" import { enter_alt_buffer, set_title, teardown } from "./term.ab" import { bail } from "./common.ab" -import { get_osname, get_system, get_cache_root } from "./platform.ab" +import { get_osname, get_system, get_cache_root, get_nix_root } from "./platform.ab" import { pull_binary } from "./resources.ab" import { build_nix } from "./builder.ab" @@ -78,25 +78,65 @@ fun get_nix() return 0 } +/// Move the contents of the user Nix store into the system Nix store. +/// +/// This reduces occupied storage space in the event that the user installed +/// Nix system-wide between uses of our script. +fun migrate_nix_store() +{ + let nix_root = get_nix_root() + + if not dir_exists("{nix_root}/nix/store"): + return 0 + + echo "Migrating Nix store to system-wide install..." + trust $nix copy --from {nix_root} --all --no-check-sigs$ + if status == 0 { + trust $chmod -R +wx {nix_root} && rm -rf {nix_root}$ + } +} + /// Launch Nix with the proper environment variables for fakedir and OpenSSL. /// /// This is required because macOS has no namespacing facility, and Nix will /// not attempt to use the system-wide CA bundle by default. This function /// addresses both issues. +/// +/// **This function does not return.** fun launch_darwin_workaround(name: Text, args: [Text]): Null { + let cache_root = get_cache_root() + let nix_root = get_nix_root() + + let nix_path = "{cache_root}/nix-static" + let fakedir_path = "{cache_root}/nix-lib/libfakedir.dylib" + trust env_var_set("FAKEDIR_PATTERN", "/nix") + trust env_var_set("FAKEDIR_TARGET", "{nix_root}/nix") + + trust $export FAKEDIR_PATTERN FAKEDIR_TARGET$ + + // We need to ensure fakedir gets propagated into child processes as well. + // Unfortunately, this requires us to disable the Nix sandbox entirely. + trust $ _NIX_TEST_NO_SANDBOX=1 \ + DYLD_INSERT_LIBRARIES="{fakedir_path}" \ + DYLD_LIBRARY_PATH="{cache_root}/nix-lib" \ + exec -a {name} {nix_path} {args}$ } /// Launch Nix with options pulled from a file's contents. /// This allows our script to be used as an interpreter, the same way /// Nix proper would. +/// +/// **This function does not return.** pub fun launch_nix_shebang(name: Text, file: Text, args: [Text]): Null { } /// Launch Nix as a regular CLI. +/// +/// **This function does not return.** pub fun launch_nix(name: Text, extras: [Text], args: [Text]): Null { get_nix() failed { echo "oh nyo" } From 60e01159bcf1bb0f9960853cf64c266a0bf8d40a Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Mon, 10 Feb 2025 13:14:35 +0100 Subject: [PATCH 12/22] Building source builders --- src/builder.ab | 59 ------------------------------ src/builders.ab | 68 +++++++++++++++++++++++++++++++++++ src/builders/autoconf.ab | 22 ++++++++++++ src/builders/boost.ab | 37 +++++++++++++++++++ src/builders/common.ab | 28 +++++++++++++++ src/builders/lowdown.ab | 18 ++++++++++ src/builders/nix.ab | 12 +++++++ src/builders/nlohmann_json.ab | 13 +++++++ src/builders/openssl.ab | 26 ++++++++++++++ src/cli.ab | 10 ++++++ src/nix.ab | 11 ++++-- src/term.ab | 40 +++++++++++++++++++-- 12 files changed, 280 insertions(+), 64 deletions(-) delete mode 100644 src/builder.ab create mode 100644 src/builders.ab create mode 100644 src/builders/autoconf.ab create mode 100644 src/builders/boost.ab create mode 100644 src/builders/common.ab create mode 100644 src/builders/lowdown.ab create mode 100644 src/builders/nix.ab create mode 100644 src/builders/nlohmann_json.ab create mode 100644 src/builders/openssl.ab diff --git a/src/builder.ab b/src/builder.ab deleted file mode 100644 index fa713ff..0000000 --- a/src/builder.ab +++ /dev/null @@ -1,59 +0,0 @@ -// Nixie © Karim Vergnes -// Licensed under GNU GPLv2 -// Functions to build Nix and its dependencies from source - -import { set_title } from "./term.ab" -import { check_deps, bail } from "./common.ab" -import { pull_source_file, pull_binary } from "./resources.ab" - -fun build_openssl() -{ - -} - - -fun build_lowdown() -{ - -} - -fun build_nlohmann_json() -{ - -} - -fun build_boost() -{ - -} - -/// Build a dependency which uses Autoconf as its build system. -/// -/// ### Arguments: -/// - `lib_name`: The library to check for with `pkg-config` -/// - `var_name`: The variable to export library locations -/// - `lib_prefix`: Where in the source tree the resulting libraries are found -/// - `inc_prefix`: Where in the source tree the build headers are found -fun build_autoconf_dep(lib_name: Text, var_name: Text, - lib_prefix: Text = "", - inc_prefix: Text = ""): Null -{ - -} - -/// Export libraries found inside the macOS SDK for building Nix -fun darwin_export_sdk() -{ - -} - -/// Build Nix and its dependencies locally, then place it in the expected location -/// in the user's cache directory. -/// -/// This process **requires**, among other things, `pkg-config` to determine -/// which dependencies are already met and avoid building them. -pub fun build_nix() -{ - check_deps(["cc", "c++", "pkg-config", "make", "flex", "bison", "perl"]) - failed { bail("Failed to build Nix from source") } -} diff --git a/src/builders.ab b/src/builders.ab new file mode 100644 index 0000000..82014ee --- /dev/null +++ b/src/builders.ab @@ -0,0 +1,68 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Functions to build Nix and its dependencies from source + +import { dir_exists } from "std/fs" +import { env_var_set } from "std/env" + +import { set_title } from "./term.ab" +import { check_deps, bail } from "./common.ab" +import { pull_source_file, pull_binary } from "./resources.ab" +import { get_osname } from "./platform.ab" + +import { build_openssl } from "./builders/openssl.ab" +import { build_lowdown } from "./builders/lowdown.ab" +import { build_nlohmann_json } from "./builders/nlohmann_json.ab" +import { build_boost } from "./builders/boost.ab" +import { build_autoconf_dep } from "./builders/autoconf.ab" +import { build_nix } from "./builders/nix.ab" + +/// Export libraries found inside the macOS SDK for building Nix +fun darwin_export_sdk() +{ + // Calling xcrun should prompt the user to install the macOS SDK. + let sdk_path = trust $xcrun --show-sdk-path$ + + if not dir_exists(sdk_path): + bail("The macOS SDK from Xcode or CommandLineTools is required to build Nix.") + + let sdk_libs = "{sdk_path}/usr/lib" + let sdk_cflags = "-I{sdk_path}/usr/include" + + trust env_var_set("LIBCURL_LIBS", sdk_libs) + trust env_var_set("LIBCURL_CFLAGS", sdk_cflags) + trust env_var_set("LIBARCHIVE_LIBS", sdk_libs) + trust env_var_set("LIBARCHIVE_CFLAGS", sdk_cflags) + trust env_var_set("OPENSSL_LIBS", sdk_libs) + trust env_var_set("OPENSSL_CFLAGS", sdk_cflags) + + trust $export LIBCURL_LIBS LIBCURL_CFLAGS \ + LIBARCHIVE_LIBS LIBARCHIVE_CFLAGS \ + OPENSSL_LIBS OPENSSL_CFLAGS$ +} + +/// Build Nix and its dependencies locally, then place it in the expected location +/// in the user's cache directory. +/// +/// This process **requires**, among other things, `pkg-config` to determine +/// which dependencies are already met and avoid building them. +pub fun try_build_nix() +{ + check_deps(["cc", "c++", "pkg-config", "make", "flex", "bison", "perl"]) + failed { bail("Missing required dependencies to build from source.") } + + if get_osname() == "Darwin": + darwin_export_sdk() + + trust env_var_set("step_total", "8") + + build_openssl()? + build_boost()? + build_nlohmann_json()? + build_lowdown()? + // pkgconf name env var lib prefix include prefix + build_autoconf_dep("libbrotlicommon", "LIBBROTLI", "", "c")? + build_autoconf_dep("libsodium", "SODIUM", "src/libsodium", "src/libsodium")? + build_autoconf_dep("libeditline", "EDITLINE", "src")? + build_nix()? +} diff --git a/src/builders/autoconf.ab b/src/builders/autoconf.ab new file mode 100644 index 0000000..9e402f7 --- /dev/null +++ b/src/builders/autoconf.ab @@ -0,0 +1,22 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Build script for Autoconf-based dependencies + +import { pkg_exists, step_title } from "./common.ab" + +/// Build a dependency which uses Autoconf as its build system. +/// +/// ### Arguments: +/// - `lib_name`: The library to check for with `pkg-config` +/// - `var_name`: The variable to export library locations +/// - `lib_prefix`: Where in the source tree the resulting libraries are found +/// - `inc_prefix`: Where in the source tree the build headers are found +pub fun build_autoconf_dep(lib_name: Text, var_name: Text, + lib_prefix: Text = "", + inc_prefix: Text = ""): Null +{ + step_title(lib_name) + + if pkg_exists(lib_name): + return null +} diff --git a/src/builders/boost.ab b/src/builders/boost.ab new file mode 100644 index 0000000..595cc0d --- /dev/null +++ b/src/builders/boost.ab @@ -0,0 +1,37 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Builder for the Boost C++ framework + +import { file_exists, dir_exists } from "std/fs" + +import { pkg_exists, step_title } from "./common.ab" + +const modules = ["chrono", "container", "context", "system", "thread"] + +/// Check that the Boost modules we need exist on the system. +/// +/// Boost does not provide a pkg-config package, nor a config program. +/// This means we have to do the information-gathering the old way, by +/// looking through /usr and /usr/local manually. +fun find_boost_libs(libs: [Text]): Bool +{ + for lib in libs { + let libname = "libboost_{lib}*" + if not ( file_exists("/usr/lib/{libname}") + or file_exists("/usr/local/lib/{libname}")): + return false + if not ( dir_exists("/usr/include/boost/{lib}") + or dir_exists("/usr/local/include/boost/{lib}")): + return false + } + + return true +} + +pub fun build_boost() +{ + step_title("boost") + + if find_boost_libs(["atomic"] + modules): + return 0 +} diff --git a/src/builders/common.ab b/src/builders/common.ab new file mode 100644 index 0000000..f3a7806 --- /dev/null +++ b/src/builders/common.ab @@ -0,0 +1,28 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Common functions for source-based builders + +import { env_var_get } from "std/env" + +import { set_title } from "../term.ab" + +let step_current = 1 + +/// Invoke `pkg-config` to assert that a given dependency exists on the system. +pub fun pkg_exists(package: Text): Bool +{ + $pkg-config {package}$ failed { + return false + } + + return true +} + +pub fun step_title(name: Text): Null +{ + let step_total = trust env_var_get("step_total") + + set_title("Building Nix: {name} ({step_current}/{step_total})") + + step_current += 1 +} diff --git a/src/builders/lowdown.ab b/src/builders/lowdown.ab new file mode 100644 index 0000000..dd4d162 --- /dev/null +++ b/src/builders/lowdown.ab @@ -0,0 +1,18 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Builder for the lowdown Markdown library + +import { pkg_exists, step_title } from "./common.ab" + +fun macos_build_post() +{ + $cc -shared -o liblowdown.1.dylib *.o$? +} + +pub fun build_lowdown() +{ + step_title("lowdown") + + if pkg_exists("lowdown"): + return 0 +} diff --git a/src/builders/nix.ab b/src/builders/nix.ab new file mode 100644 index 0000000..748d564 --- /dev/null +++ b/src/builders/nix.ab @@ -0,0 +1,12 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Builder for the Nix package manager itself + +import { file_exists, dir_exists } from "std/fs" + +import { step_title } from "./common.ab" + +pub fun build_nix() +{ + step_title("nix") +} diff --git a/src/builders/nlohmann_json.ab b/src/builders/nlohmann_json.ab new file mode 100644 index 0000000..13bc906 --- /dev/null +++ b/src/builders/nlohmann_json.ab @@ -0,0 +1,13 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Builder for NLohmann's JSON library + +import { pkg_exists, step_title } from "./common.ab" + +pub fun build_nlohmann_json() +{ + step_title("nlohmann_json") + + if pkg_exists("nlohmann_json"): + return 0 +} diff --git a/src/builders/openssl.ab b/src/builders/openssl.ab new file mode 100644 index 0000000..6842a02 --- /dev/null +++ b/src/builders/openssl.ab @@ -0,0 +1,26 @@ +// Nixie © Karim Vergnes +// Licensed under GNU GPLv2 +// Builder for OpenSSL + +import { env_var_test } from "std/env" + +import { pkg_exists, step_title } from "./common.ab" + +/// Scans through and builds generated headers required for building OpenSSL. +/// +/// For some reason the OpenSSL Makefile does not specify dependencies to those +/// targets, meaning we cannot simply `make libcrypto.so`. +fun make_headers() +{ + +} + +pub fun build_openssl() +{ + step_title("libcrypto") + + if pkg_exists("libcrypto"): + return 0 + if env_var_test("OPENSSL_LIBS") and env_var_test("OPENSSL_CFLAGS"): + return 0 +} diff --git a/src/cli.ab b/src/cli.ab index 509006a..080fda3 100644 --- a/src/cli.ab +++ b/src/cli.ab @@ -69,6 +69,15 @@ fun opt_ignore_system() trust env_var_set("nosystem", "1") } +//TODO: debug function, remove! +fun no_precompiled() +{ + echo_warning("Ignoring precompiled binaries for testing purposes.") + echo_warning("This implies --nixie-ignore-system.") + trust env_var_set("nobins", "1") + trust env_var_set("nosystem", "1") +} + fun notfound(cmd) { let self = trust env_var_get("0") @@ -84,6 +93,7 @@ fun eval_cmd(cmd) cmd == "extract": cmd_extract() cmd == "cleanup": cmd_cleanup() cmd == "ignore-system": opt_ignore_system() + cmd == "no-precompiled":no_precompiled() else: notfound(cmd) } } diff --git a/src/nix.ab b/src/nix.ab index e966be7..0af3db9 100644 --- a/src/nix.ab +++ b/src/nix.ab @@ -9,7 +9,7 @@ import { enter_alt_buffer, set_title, teardown } from "./term.ab" import { bail } from "./common.ab" import { get_osname, get_system, get_cache_root, get_nix_root } from "./platform.ab" import { pull_binary } from "./resources.ab" -import { build_nix } from "./builder.ab" +import { try_build_nix } from "./builders.ab" /// Determine whether Nix is installed system-wide. Nixie can launch the /// system-wide Nix when it is available, using the script's bundled options. @@ -67,9 +67,16 @@ fun get_nix() return 0 } + //TODO: Debug! Remove before release! + if env_var_test("nobins") { + trust try_build_nix() + teardown() + fail status + } + // Try to extract or download Nix pull_binary("nix.{system}", nix_path) failed { - trust build_nix() + trust try_build_nix() teardown() fail status } diff --git a/src/term.ab b/src/term.ab index ba17898..f38b2f0 100644 --- a/src/term.ab +++ b/src/term.ab @@ -2,21 +2,55 @@ // Licensed under GNU GPLv2 // Utilities to manipulate Xterm features -let tsl = trust $tput tsl$ //TODO: improve logic to match xterm+sl -let fsl = trust $tput fsl$ //TODO: as above +import { env_var_get } from "std/env" +import { starts_with } from "std/text" + +let tsl = trust $tput tsl$ +let fsl = trust $tput fsl$ let smcup = trust $tput smcup$ let rmcup = trust $tput rmcup$ +let TERM = trust env_var_get("TERM") + +fun can_set_title(): Bool +{ + let has_statusline = true + + $tput hs$ failed { + // For some reason, default xterm won't communicate statusline. + // We need to set TERM=xterm+sl to get the info. + if starts_with(TERM, "xterm") { + trust $TERM=xterm+sl tput hs$ + if status == 0 { + tsl = trust $TERM=xterm+sl tput tsl$ + fsl = trust $TERM=xterm+sl tput fsl$ + } else { + has_statusline = false + } + } else { + has_statusline = false + } + } + + return has_statusline +} + +/// Set the title for the current window, if possible. +/// +/// Does nothing otherwise. pub fun set_title(title: Text = ""): Null { - echo "{tsl}{title} {fsl}" + if can_set_title(): + echo tsl + title + fsl } +/// Create and enter an alt-buffer in the terminal. pub fun enter_alt_buffer(): Null { echo smcup } +/// Close the alt-buffer in the terminal. pub fun exit_alt_buffer(): Null { echo rmcup From 6a0047350924b00fe20de51520559070cc533a24 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Sat, 15 Feb 2025 13:06:37 +0100 Subject: [PATCH 13/22] Prototype Nix command executor --- src/main.ab | 2 +- src/nix.ab | 85 +++++++++++++++++++++++++++++++++++++------------ src/platform.ab | 7 +++- 3 files changed, 72 insertions(+), 22 deletions(-) diff --git a/src/main.ab b/src/main.ab index 357a9fa..bf089ac 100644 --- a/src/main.ab +++ b/src/main.ab @@ -62,5 +62,5 @@ main(cmdl) let self = array_shift(args) - launch_nix(self, ["lol"], args) + launch_nix(self, args) } diff --git a/src/nix.ab b/src/nix.ab index 0af3db9..2f5e194 100644 --- a/src/nix.ab +++ b/src/nix.ab @@ -4,6 +4,8 @@ import { file_exists, dir_exists, dir_create } from "std/fs" import { env_var_test, env_var_set } from "std/env" +import { split, slice, starts_with, ends_with } from "std/text" +import { array_last, array_shift } from "std/array" import { enter_alt_buffer, set_title, teardown } from "./term.ab" import { bail } from "./common.ab" @@ -53,12 +55,7 @@ fun get_nix() if osname == "Darwin" and not file_exists(fakedir_path) { dir_create("{cache_root}/nix-lib") - trust pull_binary("libfakedir.dylib", fakedir_path) - } - - if is_nix_installed() { - teardown() - return 0 + pull_binary("libfakedir.dylib", fakedir_path)? } // We already have static Nix @@ -81,6 +78,8 @@ fun get_nix() fail status } + trust $chmod +x {nix_path}$ + teardown() return 0 } @@ -110,12 +109,11 @@ fun migrate_nix_store() /// addresses both issues. /// /// **This function does not return.** -fun launch_darwin_workaround(name: Text, args: [Text]): Null +fun launch_darwin_workaround(name: Text, nix_path: Text, args: [Text]): Null { let cache_root = get_cache_root() let nix_root = get_nix_root() - let nix_path = "{cache_root}/nix-static" let fakedir_path = "{cache_root}/nix-lib/libfakedir.dylib" trust env_var_set("FAKEDIR_PATTERN", "/nix") @@ -131,27 +129,74 @@ fun launch_darwin_workaround(name: Text, args: [Text]): Null exec -a {name} {nix_path} {args}$ } -/// Launch Nix with options pulled from a file's contents. +/// Launch Nix with options pulled from a file's contents. +/// /// This allows our script to be used as an interpreter, the same way /// Nix proper would. /// -/// **This function does not return.** -pub fun launch_nix_shebang(name: Text, file: Text, args: [Text]): Null +/// ### Arguments: +/// - `nix_path`: Path to the Nix executable to use (local or installed) +/// - `file`: The filename to interpret with Nix +/// - `extras`: Internal options to append to the Nix command line +/// - `args`: Command line options to pass to the interpreted file +pub fun launch_nix_shebang(nix_path: Text, file: Text, extras: [Text], args: [Text]): Null { + let shebang = "" + for i, line in lines(file) { + if i == 1 { + shebang = line + break + } + } + // No second shebang, not our business. + if not starts_with(shebang, "#!"): + return null + let bang_args = split(slice(shebang, 2), " ") + let name = array_shift(bang_args) + + // Find and inject script options into --command + for i, arg in bang_args { + if arg == "-i" { + bang_args[i] = "--command" + bang_args[i+1] = "{bang_args[i+1]} {file} {args}" + break + } + } + trust $exec -a {name} {nix_path} "{extras}" "{bang_args}"$ } /// Launch Nix as a regular CLI. /// /// **This function does not return.** -pub fun launch_nix(name: Text, extras: [Text], args: [Text]): Null +pub fun launch_nix(self: Text, args: [Text]): Null { - get_nix() failed { echo "oh nyo" } - - echo "Would launch Nix as {name}" - echo "Internal options: {extras}" - echo "Command line options: {args}" - echo "" - echo "That's all we have for now, thanks for playing!" - exit 0 + let cache_root = get_cache_root() + let nix_root = get_nix_root() + let osname = get_osname() + + let nix_path = "{cache_root}/nix-static" + + let extras = [Text] + let name = array_last(split(self, "/")) + + if is_nix_installed() { + migrate_nix_store() + nix_path = "nix" + } else { + get_nix() failed { + bail("Failed to obtain Nix. Check your internet connection.") + } + } + + if file_exists(args[0]) and not ends_with(args[0], ".nix") { + let args_shebang = args + array_shift(args_shebang) + launch_nix_shebang(nix_path, args[0], extras, args_shebang) + } + + if osname == "Darwin" and not is_nix_installed(): + launch_darwin_workaround(name, nix_path, extras + args) + else: + trust $exec -a {name} {nix_path} "{extras}" "{args}"$ } diff --git a/src/platform.ab b/src/platform.ab index 832d936..82a028e 100644 --- a/src/platform.ab +++ b/src/platform.ab @@ -17,7 +17,12 @@ pub fun get_osname(): Text /// Return the current machine architecture, as reported by `uname -s`. pub fun get_machine(): Text { - return trust $uname -m$ + let machine = trust $uname -m$ + + // Normalize to Linux 'aarch64' name + if machine == "arm64": + return "aarch64" + return machine } /// Return the combined system-string, in the format `osname.machine`. From 99a6f49a98880f7f29efc77007050f750b005555 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Sat, 15 Feb 2025 16:00:09 +0100 Subject: [PATCH 14/22] Finailized Nix launcher --- src/main.ab | 12 +++++++++- src/nix.ab | 68 ++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 68 insertions(+), 12 deletions(-) diff --git a/src/main.ab b/src/main.ab index bf089ac..9956eb2 100644 --- a/src/main.ab +++ b/src/main.ab @@ -14,10 +14,12 @@ let NIXIE_VERSION = "" let args = [Text] -import { env_var_get } from "std/env" +import { env_var_get, env_var_set } from "std/env" import { array_shift } from "std/array" +import { file_exists } from "std/fs" import { bail, untar, check_deps } from "./common.ab" +import { get_osname } from "./platform.ab" import { catch_args } from "./cli.ab" import { launch_nix } from "./nix.ab" @@ -62,5 +64,13 @@ main(cmdl) let self = array_shift(args) + // Check for alternate OpenSSL/LibreSSL certificate paths (fixes #6) + if { + file_exists("/etc/pki/tls/certs/ca-bundle.crt"): + trust env_var_set("NIX_SSL_CERT_FILE", "/etc/pki/tls/certs/ca-bundle.crt") + get_osname() == "Darwin": + trust env_var_set("NIX_SSL_CERT_FILE", "/etc/ssl/cert.pem") + } + launch_nix(self, args) } diff --git a/src/nix.ab b/src/nix.ab index 2f5e194..358c659 100644 --- a/src/nix.ab +++ b/src/nix.ab @@ -2,10 +2,10 @@ // Licensed under GNU GPLv2 // Final-step Nix invocation methods -import { file_exists, dir_exists, dir_create } from "std/fs" -import { env_var_test, env_var_set } from "std/env" -import { split, slice, starts_with, ends_with } from "std/text" -import { array_last, array_shift } from "std/array" +import { file_exists, dir_exists, dir_create } from "std/fs" +import { env_var_test, env_var_set, env_var_get } from "std/env" +import { split, slice, starts_with, ends_with } from "std/text" +import { array_last, array_shift } from "std/array" import { enter_alt_buffer, set_title, teardown } from "./term.ab" import { bail } from "./common.ab" @@ -90,7 +90,11 @@ fun get_nix() /// Nix system-wide between uses of our script. fun migrate_nix_store() { - let nix_root = get_nix_root() + let nix_root = "" + + nix_root = $readlink -f {get_nix_root()}$ failed { + nix_root = get_nix_root() + } if not dir_exists("{nix_root}/nix/store"): return 0 @@ -102,6 +106,40 @@ fun migrate_nix_store() } } +/// Provide a list of extra arguments for bundled Nix channels, after +/// extracting said channels from the bundled archive. +fun unpack_channels(): [Text] +{ + let args = [Text] +} + +/// Provide a list of extra arguments depending on the script's features +/// and the runtime environment. +fun populate_extras(): [Text] +{ + let args = [Text] + + let EXTRA_FEATURES = trust env_var_get("EXTRA_FEATURES") + let EXTRA_SUBSTITUTERS = trust env_var_get("EXTRA_SUBSTITUTERS") + let EXTRA_TRUSTED_PUBLIC_KEYS = trust env_var_get("EXTRA_TRUSTED_PUBLIC_KEYS") + + let nix_root = get_nix_root() + + if EXTRA_FEATURES != "": + args += [ "--extra-experimental-features", EXTRA_FEATURES ] + + if EXTRA_SUBSTITUTERS != "": + args += [ "--extra-substituters", EXTRA_SUBSTITUTERS ] + + if EXTRA_TRUSTED_PUBLIC_KEYS != "": + args += [ "--extra-trusted-public-keys", EXTRA_TRUSTED_PUBLIC_KEYS ] + + if get_osname() != "Darwin" and not is_nix_installed(): + args += [ "--store", trust $readlink -f {nix_root}$ ] + + return args +} + /// Launch Nix with the proper environment variables for fakedir and OpenSSL. /// /// This is required because macOS has no namespacing facility, and Nix will @@ -126,7 +164,7 @@ fun launch_darwin_workaround(name: Text, nix_path: Text, args: [Text]): Null trust $ _NIX_TEST_NO_SANDBOX=1 \ DYLD_INSERT_LIBRARIES="{fakedir_path}" \ DYLD_LIBRARY_PATH="{cache_root}/nix-lib" \ - exec -a {name} {nix_path} {args}$ + exec -a {name} {nix_path} "{args}"$ } /// Launch Nix with options pulled from a file's contents. @@ -139,7 +177,7 @@ fun launch_darwin_workaround(name: Text, nix_path: Text, args: [Text]): Null /// - `file`: The filename to interpret with Nix /// - `extras`: Internal options to append to the Nix command line /// - `args`: Command line options to pass to the interpreted file -pub fun launch_nix_shebang(nix_path: Text, file: Text, extras: [Text], args: [Text]): Null +fun launch_nix_shebang(nix_path: Text, file: Text, extras: [Text], args: [Text]): Null { let shebang = "" for i, line in lines(file) { @@ -163,7 +201,11 @@ pub fun launch_nix_shebang(nix_path: Text, file: Text, extras: [Text], args: [Te break } } - trust $exec -a {name} {nix_path} "{extras}" "{bang_args}"$ + + if get_osname() == "Darwin" and not is_nix_installed(): + launch_darwin_workaround(name, nix_path, extras + bang_args) + else: + trust $exec -a {name} {nix_path} "{extras}" "{bang_args}"$ } /// Launch Nix as a regular CLI. @@ -173,13 +215,15 @@ pub fun launch_nix(self: Text, args: [Text]): Null { let cache_root = get_cache_root() let nix_root = get_nix_root() - let osname = get_osname() let nix_path = "{cache_root}/nix-static" - let extras = [Text] + let extras = populate_extras() let name = array_last(split(self, "/")) + if starts_with(name, "nix-"): + extras += unpack_channels() + if is_nix_installed() { migrate_nix_store() nix_path = "nix" @@ -189,13 +233,15 @@ pub fun launch_nix(self: Text, args: [Text]): Null } } + trust $export NIX_SSL_CERT_FILE$ + if file_exists(args[0]) and not ends_with(args[0], ".nix") { let args_shebang = args array_shift(args_shebang) launch_nix_shebang(nix_path, args[0], extras, args_shebang) } - if osname == "Darwin" and not is_nix_installed(): + if get_osname() == "Darwin" and not is_nix_installed(): launch_darwin_workaround(name, nix_path, extras + args) else: trust $exec -a {name} {nix_path} "{extras}" "{args}"$ From c615416e12eec5839d2b6eb7dffee5f267a79385 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Sun, 16 Feb 2025 11:28:18 +0100 Subject: [PATCH 15/22] Added channel unpacking --- src/cli.ab | 1 - src/common.ab | 18 ++++++++++++++++-- src/nix.ab | 40 +++++++++++++++++++++++++++++++--------- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/src/cli.ab b/src/cli.ab index 080fda3..b288c7a 100644 --- a/src/cli.ab +++ b/src/cli.ab @@ -69,7 +69,6 @@ fun opt_ignore_system() trust env_var_set("nosystem", "1") } -//TODO: debug function, remove! fun no_precompiled() { echo_warning("Ignoring precompiled binaries for testing purposes.") diff --git a/src/common.ab b/src/common.ab index 8ee55b4..ab672d0 100644 --- a/src/common.ab +++ b/src/common.ab @@ -2,8 +2,8 @@ // Licensed under GNU GPLv2 // Common utility functions -import { echo_error, env_var_get, is_command } from "std/env" -import { starts_with, text_contains } from "std/text" +import { echo_error, env_var_get, is_command } from "std/env" +import { starts_with, text_contains, parse_number } from "std/text" import { exit_alt_buffer, set_title } from "./term.ab" @@ -30,6 +30,20 @@ if text_contains(me, "/") { echo_error("This script must be run from an absolute or relative path.") } +/// Checks that the left-hand path exists and is newer than the right-hand path. +/// False otherwise. +pub fun exists_newer(left: Text, right: Text): Bool +{ + let left_time = parse_number(trust $stat -c %W {left}$) failed { + return false // Untrustworthy stat, assume worst + } + let right_time = parse_number(trust $stat -c %W {right}$) failed { + return false + } + + return left_time >= right_time +} + /// Exit alt-buffer, quit script, kill ourselves. At this point something has gone /// catastrophically wrong and the script MUST stop. /// diff --git a/src/nix.ab b/src/nix.ab index 358c659..dc05199 100644 --- a/src/nix.ab +++ b/src/nix.ab @@ -8,8 +8,9 @@ import { split, slice, starts_with, ends_with } from "std/text" import { array_last, array_shift } from "std/array" import { enter_alt_buffer, set_title, teardown } from "./term.ab" -import { bail } from "./common.ab" -import { get_osname, get_system, get_cache_root, get_nix_root } from "./platform.ab" +import { bail, untar, exists_newer, get_self } from "./common.ab" +import { get_osname, get_system, get_cache_root + , get_nix_root, get_repo_root } from "./platform.ab" import { pull_binary } from "./resources.ab" import { try_build_nix } from "./builders.ab" @@ -53,29 +54,36 @@ fun get_nix() // Cleanly exit alt-buffer if hit with ^C trust $trap "{nameof teardown}; exit 1" SIGKILL SIGTERM SIGINT SIGABRT$ + // Unpack fakedir as needed if osname == "Darwin" and not file_exists(fakedir_path) { dir_create("{cache_root}/nix-lib") - pull_binary("libfakedir.dylib", fakedir_path)? + pull_binary("libfakedir.dylib", fakedir_path) failed { + teardown() + fail 1 + } } + // Update it if applicable + if osname == "Darwin" and not exists_newer(fakedir_path, get_self()): + trust pull_binary("libfakedir.dylib", fakedir_path) + // We already have static Nix if file_exists(nix_path) { teardown() return 0 } - //TODO: Debug! Remove before release! if env_var_test("nobins") { trust try_build_nix() teardown() - fail status + fail 1 } // Try to extract or download Nix pull_binary("nix.{system}", nix_path) failed { trust try_build_nix() teardown() - fail status + fail 1 } trust $chmod +x {nix_path}$ @@ -108,9 +116,23 @@ fun migrate_nix_store() /// Provide a list of extra arguments for bundled Nix channels, after /// extracting said channels from the bundled archive. -fun unpack_channels(): [Text] +fun unpack_channels() { - let args = [Text] + let repo_root = get_repo_root() + + if exists_newer("{repo_root}/.nixie/channels", get_self()) { + let NIX_PATH = trust env_var_get("NIX_PATH") + trust env_var_set("NIX_PATH", "{repo_root}/.nixie/channels:{NIX_PATH}") + trust $export NIX_PATH$ + return 0 + } + echo "Unpacking Nix channels, hang tight..." + + dir_create("{repo_root}/.nixie") + untar("channels -C {repo_root}/.nixie") failed { + // Leave an empty directory to trip the time check above + trust $mkdir {repo_root}/.nixie/channels$ + } } /// Provide a list of extra arguments depending on the script's features @@ -222,7 +244,7 @@ pub fun launch_nix(self: Text, args: [Text]): Null let name = array_last(split(self, "/")) if starts_with(name, "nix-"): - extras += unpack_channels() + trust unpack_channels() if is_nix_installed() { migrate_nix_store() From 7443cf1f1f4c4524d7302344e8ebfa88160b5054 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Sun, 23 Feb 2025 18:51:21 +0100 Subject: [PATCH 16/22] Added command substitution launcher --- src/cli.ab | 14 +++++++------- src/nix.ab | 52 +++++++++++++++++++++++++++++++++++++++++++++------- src/term.ab | 6 +++++- 3 files changed, 57 insertions(+), 15 deletions(-) diff --git a/src/cli.ab b/src/cli.ab index b288c7a..9dcc915 100644 --- a/src/cli.ab +++ b/src/cli.ab @@ -69,7 +69,7 @@ fun opt_ignore_system() trust env_var_set("nosystem", "1") } -fun no_precompiled() +fun opt_no_precompiled() { echo_warning("Ignoring precompiled binaries for testing purposes.") echo_warning("This implies --nixie-ignore-system.") @@ -87,12 +87,12 @@ fun notfound(cmd) fun eval_cmd(cmd) { if { - cmd == "help": cmd_help() - cmd == "print-config": cmd_print_config() - cmd == "extract": cmd_extract() - cmd == "cleanup": cmd_cleanup() - cmd == "ignore-system": opt_ignore_system() - cmd == "no-precompiled":no_precompiled() + cmd == "help": cmd_help() + cmd == "print-config": cmd_print_config() + cmd == "extract": cmd_extract() + cmd == "cleanup": cmd_cleanup() + cmd == "ignore-system": opt_ignore_system() + cmd == "no-precompiled": opt_no_precompiled() else: notfound(cmd) } } diff --git a/src/nix.ab b/src/nix.ab index dc05199..78d6261 100644 --- a/src/nix.ab +++ b/src/nix.ab @@ -58,7 +58,7 @@ fun get_nix() if osname == "Darwin" and not file_exists(fakedir_path) { dir_create("{cache_root}/nix-lib") pull_binary("libfakedir.dylib", fakedir_path) failed { - teardown() + teardown(true) fail 1 } } @@ -74,16 +74,18 @@ fun get_nix() } if env_var_test("nobins") { - trust try_build_nix() - teardown() - fail 1 + try_build_nix() failed { + teardown(true) + fail 1 + } } // Try to extract or download Nix pull_binary("nix.{system}", nix_path) failed { - trust try_build_nix() - teardown() - fail 1 + try_build_nix() failed { + teardown(true) + fail 1 + } } trust $chmod +x {nix_path}$ @@ -189,6 +191,38 @@ fun launch_darwin_workaround(name: Text, nix_path: Text, args: [Text]): Null exec -a {name} {nix_path} "{args}"$ } +/// Launch the command referred to by name from within a Nix shell. +/// +/// This allows our script to be used as an alias for developer utilities. +/// +/// ### Arguments: +/// - `nix_path`: Path to the Nix executable to use (local or installed) +/// - `cmd`: The name our script was called as +/// - `extras`: Internal options to append to the Nix command line +/// - `args`: Command line options to pass to the command +fun launch_shell_command(nix_path: Text, cmd: Text, extras: [Text], args: [Text]): Null +{ + let repo_root = get_repo_root() + let pwd = trust env_var_get("PWD") + let name = "nix-shell" + let shell_cmd = [ "{repo_root}/shell.nix", "--command", "{cmd} {args}" ] + + if file_exists("{pwd}/flake.nix") { + name = "nix" + shell_cmd = [ "develop", pwd, "-c", cmd ] + args + } else: if file_exists("{repo_root}/flake.nix") { + name = "nix" + shell_cmd = [ "develop", repo_root, "-c", cmd ] + args + } else: if file_exists("{pwd}/shell.nix") { + shell_cmd = [ "{pwd}/shell.nix", "--command", "{cmd} {args}" ] + } + + if get_osname() == "Darwin" and not is_nix_installed(): + launch_darwin_workaround(name, nix_path, extras + shell_cmd) + else: + trust $exec -a {name} {nix_path} "{extras}" "{shell_cmd}"$ +} + /// Launch Nix with options pulled from a file's contents. /// /// This allows our script to be used as an interpreter, the same way @@ -263,6 +297,10 @@ pub fun launch_nix(self: Text, args: [Text]): Null launch_nix_shebang(nix_path, args[0], extras, args_shebang) } + if not starts_with(name, "nix") { + launch_shell_command(nix_path, name, extras, args) + } + if get_osname() == "Darwin" and not is_nix_installed(): launch_darwin_workaround(name, nix_path, extras + args) else: diff --git a/src/term.ab b/src/term.ab index f38b2f0..a36ec6f 100644 --- a/src/term.ab +++ b/src/term.ab @@ -57,8 +57,12 @@ pub fun exit_alt_buffer(): Null } /// Shorthand to exit alt-buffer and clear terminal title. -pub fun teardown(): Null +pub fun teardown(failure: Bool = false): Null { + if failure { + echo "Press any key to continue..." + trust $read -k 1 -s$ + } set_title() exit_alt_buffer() } From ba3baa96750b76759177a54a8c7aa8dbdbb44ba9 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Sun, 23 Feb 2025 20:44:31 +0100 Subject: [PATCH 17/22] Implemented dependency builders --- sources/Makefile | 4 +-- src/builders.ab | 13 +++++++--- src/builders/autoconf.ab | 24 ++++++++++++++++-- src/builders/boost.ab | 34 +++++++++++++++++++++++-- src/builders/common.ab | 11 ++++++++ src/builders/lowdown.ab | 32 +++++++++++++++++++++++- src/builders/nix.ab | 47 +++++++++++++++++++++++++++++++++-- src/builders/nlohmann_json.ab | 14 ++++++++++- src/builders/openssl.ab | 42 +++++++++++++++++++++++++++++-- src/resources.ab | 4 ++- src/term.ab | 4 +-- 11 files changed, 211 insertions(+), 18 deletions(-) diff --git a/sources/Makefile b/sources/Makefile index 6a9c75c..b7ce1f5 100644 --- a/sources/Makefile +++ b/sources/Makefile @@ -7,10 +7,10 @@ TAR := tar WORKDIR := work WGET := wget -BOOST_VER := 1.81.0 +BOOST_VER := 1.87.0 BOOST_ARCHIVE := $(WORKDIR)/boost.tar.bz2 -BOOST_ADD_HEADERS = core,utility,io,system,thread,context,lexical_cast,config,format,coroutine2,container,chrono,atomic,predef,move,assert,detail,type_traits,intrusive,mpl,date_time,bind,align,preprocessor,ratio,exception,smart_ptr,numeric,functional,container_hash,describe,tuple,iterator,function,integer,type_index,algorithm,range,concept,optional +BOOST_ADD_HEADERS = core,utility,io,system,thread,context,lexical_cast,config,format,coroutine2,container,chrono,atomic,predef,move,assert,detail,type_traits,intrusive,mpl,date_time,bind,align,preprocessor,ratio,exception,smart_ptr,numeric,functional,container_hash,describe,tuple,iterator,function,integer,type_index,algorithm,range,concept,optional,mp11 BOOST_ADD_MODULES = system,thread,context,format,coroutine2,container,chrono,atomic,optional boost-shaved.tar.gz: $(BOOST_ARCHIVE) diff --git a/src/builders.ab b/src/builders.ab index 82014ee..b771ab4 100644 --- a/src/builders.ab +++ b/src/builders.ab @@ -2,13 +2,15 @@ // Licensed under GNU GPLv2 // Functions to build Nix and its dependencies from source -import { dir_exists } from "std/fs" -import { env_var_set } from "std/env" +import { dir_exists, dir_create } from "std/fs" +import { env_var_set } from "std/env" import { set_title } from "./term.ab" import { check_deps, bail } from "./common.ab" import { pull_source_file, pull_binary } from "./resources.ab" -import { get_osname } from "./platform.ab" +import { get_osname, get_cache_root } from "./platform.ab" + +import { get_source_root } from "./builders/common.ab" import { build_openssl } from "./builders/openssl.ab" import { build_lowdown } from "./builders/lowdown.ab" @@ -48,6 +50,8 @@ fun darwin_export_sdk() /// which dependencies are already met and avoid building them. pub fun try_build_nix() { + let cache_root = get_cache_root() + check_deps(["cc", "c++", "pkg-config", "make", "flex", "bison", "perl"]) failed { bail("Missing required dependencies to build from source.") } @@ -56,6 +60,9 @@ pub fun try_build_nix() trust env_var_set("step_total", "8") + dir_create(get_source_root()) + dir_create("{cache_root}/nix-lib") + build_openssl()? build_boost()? build_nlohmann_json()? diff --git a/src/builders/autoconf.ab b/src/builders/autoconf.ab index 9e402f7..a30a45c 100644 --- a/src/builders/autoconf.ab +++ b/src/builders/autoconf.ab @@ -2,7 +2,12 @@ // Licensed under GNU GPLv2 // Build script for Autoconf-based dependencies -import { pkg_exists, step_title } from "./common.ab" +import { env_var_set } from "std/env" + +import { pull_source_file } from "../resources.ab" +import { get_cache_root } from "../platform.ab" + +import { pkg_exists, step_title, get_source_root } from "./common.ab" /// Build a dependency which uses Autoconf as its build system. /// @@ -13,10 +18,25 @@ import { pkg_exists, step_title } from "./common.ab" /// - `inc_prefix`: Where in the source tree the build headers are found pub fun build_autoconf_dep(lib_name: Text, var_name: Text, lib_prefix: Text = "", - inc_prefix: Text = ""): Null + inc_prefix: Text = ""): Null? { + let source_root = get_source_root() + let cache_root = get_cache_root() + + let my_source = "{source_root}/{lib_name}" + step_title(lib_name) if pkg_exists(lib_name): return null + + pull_source_file(lib_name, my_source)? + + $(cd {my_source} && ./configure && make)$? + + $cp {my_source}/{lib_prefix}/.libs/* {cache_root}/nix-lib/$? + + trust env_var_set("{var_name}_LIBS", "{cache_root}/nix-lib") + trust env_var_set("{var_name}_CFLAGS", "-I{my_source}/{inc_prefix}/include") + trust $export {var_name}_LIBS {var_name}_CFLAGS$ } diff --git a/src/builders/boost.ab b/src/builders/boost.ab index 595cc0d..6fe31d6 100644 --- a/src/builders/boost.ab +++ b/src/builders/boost.ab @@ -2,9 +2,12 @@ // Licensed under GNU GPLv2 // Builder for the Boost C++ framework -import { file_exists, dir_exists } from "std/fs" +import { file_exists, dir_exists } from "std/fs" +import { env_var_set } from "std/env" -import { pkg_exists, step_title } from "./common.ab" +import { pull_source_file } from "../resources.ab" + +import { pkg_exists, step_title, get_source_root } from "./common.ab" const modules = ["chrono", "container", "context", "system", "thread"] @@ -28,10 +31,37 @@ fun find_boost_libs(libs: [Text]): Bool return true } +fun build_boost_inner() +{ + let args = [ "--static" ] + + for mod in modules { + args += [ "--with-{mod}" ] + } + + $./bootstrap.sh$? + $./b2 "{args}"$? +} + pub fun build_boost() { + let source_root = get_source_root() + step_title("boost") if find_boost_libs(["atomic"] + modules): return 0 + + pull_source_file("boost", "{source_root}/boost")? + + $(cd {source_root}/boost && {nameof build_boost_inner})$? + + trust env_var_set("BOOST_ROOT", "{source_root}/boost") + trust $export BOOST_ROOT$ +} + +// Quick mark-as-used workaround +if false { + exit 300 + trust build_boost_inner() } diff --git a/src/builders/common.ab b/src/builders/common.ab index f3a7806..214ccbc 100644 --- a/src/builders/common.ab +++ b/src/builders/common.ab @@ -3,8 +3,10 @@ // Common functions for source-based builders import { env_var_get } from "std/env" +import { dir_create } from "std/fs" import { set_title } from "../term.ab" +import { get_repo_root } from "../platform.ab" let step_current = 1 @@ -22,7 +24,16 @@ pub fun step_title(name: Text): Null { let step_total = trust env_var_get("step_total") + trust $clear$ + set_title("Building Nix: {name} ({step_current}/{step_total})") step_current += 1 } + +pub fun get_source_root(): Text +{ + let repo_root = get_repo_root() + + return "{repo_root}/.nixie/sources" +} diff --git a/src/builders/lowdown.ab b/src/builders/lowdown.ab index dd4d162..7e957b4 100644 --- a/src/builders/lowdown.ab +++ b/src/builders/lowdown.ab @@ -2,17 +2,47 @@ // Licensed under GNU GPLv2 // Builder for the lowdown Markdown library -import { pkg_exists, step_title } from "./common.ab" +import { env_var_set } from "std/env" + +import { pull_source_file } from "../resources.ab" +import { get_osname, get_cache_root } from "../platform.ab" + +import { pkg_exists, step_title, get_source_root } from "./common.ab" fun macos_build_post() { $cc -shared -o liblowdown.1.dylib *.o$? } +fun build_lowdown_inner() +{ + $./configure$? + $make$? + + if get_osname() == "Darwin": + macos_build_post()? +} + pub fun build_lowdown() { + let source_root = get_source_root() + let cache_root = get_cache_root() + step_title("lowdown") if pkg_exists("lowdown"): return 0 + + pull_source_file("lowdown", "{source_root}/lowdown")? + + $(cd {source_root}/lowdown && {nameof build_lowdown_inner})$? + + trust env_var_set("LOWDOWN_LIBS", "{cache_root}/nix-lib") + trust env_var_set("LOWDOWN_CFLAGS", "-I{source_root}/lowdown") +} + +// Quick mark-as-used workaround +if false { + exit 300 + trust build_lowdown_inner() } diff --git a/src/builders/nix.ab b/src/builders/nix.ab index 748d564..665f30b 100644 --- a/src/builders/nix.ab +++ b/src/builders/nix.ab @@ -2,11 +2,54 @@ // Licensed under GNU GPLv2 // Builder for the Nix package manager itself -import { file_exists, dir_exists } from "std/fs" +import { file_exists, dir_exists, dir_create } from "std/fs" -import { step_title } from "./common.ab" +import { pull_source_file } from "../resources.ab" +import { get_cache_root } from "../platform.ab" + +import { step_title, get_source_root } from "./common.ab" + +fun build_nix_inner() +{ + let source_root = get_source_root() + let venv = "{source_root}/nix/venv" + + $mkdir build && cd build$? + + ${venv}/bin/meson setup -Dlibstore:seccomp-sandboxing=disabled \ + -Dlibcmd:readline-flavor=editline \ + -Dlibexpr:gc=disabled \ + -Dlibutil:cpuid=disabled \ + -Ddoc-gen=false \ + -Dunit-tests=false \ + -Dbindings=false$? + + ${venv}/bin/ninja$? +} pub fun build_nix() { + let source_root = get_source_root() + let cache_root = get_cache_root() + + let venv = "{source_root}/nix/venv" + step_title("nix") + + pull_source_file("nix", "{source_root}/nix")? + + $python3 -m venv --system-site-packages "{venv}"$? + + ${venv}/bin/pip install meson ninja$? + + $(cd {source_root}/nix && {nameof build_nix_inner})$? + + trust mv "{source_root}/nix/src/nix/nix" "{cache_root}/nix-static" +} + + +// Quick mark-as-used workaround +if false { + exit 300 + trust build_nix_inner() } diff --git a/src/builders/nlohmann_json.ab b/src/builders/nlohmann_json.ab index 13bc906..1a337b6 100644 --- a/src/builders/nlohmann_json.ab +++ b/src/builders/nlohmann_json.ab @@ -2,12 +2,24 @@ // Licensed under GNU GPLv2 // Builder for NLohmann's JSON library -import { pkg_exists, step_title } from "./common.ab" +import { env_var_set } from "std/env" + +import { pull_source_file } from "../resources.ab" + +import { pkg_exists, step_title, get_source_root } from "./common.ab" pub fun build_nlohmann_json() { + let source_root = get_source_root() + step_title("nlohmann_json") if pkg_exists("nlohmann_json"): return 0 + + pull_source_file("nlohmann_json", "{source_root}/nlohmann_json")? + + trust env_var_set("NLOHMANN_JSON_LIBS", "{source_root}/nlohmann_json/single_include") + trust env_var_set("NLOHMANN_JSON_CFLAGS", "{source_root}/nlohmann_json/single_include") + trust $export NLOHMANN_JSON_LIBS NLOHMANN_JSON_CFLAGS$ } diff --git a/src/builders/openssl.ab b/src/builders/openssl.ab index 6842a02..b9c5deb 100644 --- a/src/builders/openssl.ab +++ b/src/builders/openssl.ab @@ -2,9 +2,13 @@ // Licensed under GNU GPLv2 // Builder for OpenSSL -import { env_var_test } from "std/env" +import { env_var_test, env_var_set } from "std/env" +import { split_lines } from "std/text" -import { pkg_exists, step_title } from "./common.ab" +import { pull_source_file } from "../resources.ab" +import { get_dll_ext, get_cache_root } from "../platform.ab" + +import { pkg_exists, step_title, get_source_root } from "./common.ab" /// Scans through and builds generated headers required for building OpenSSL. /// @@ -12,15 +16,49 @@ import { pkg_exists, step_title } from "./common.ab" /// targets, meaning we cannot simply `make libcrypto.so`. fun make_headers() { + for header in split_lines(trust $grep ".*\\.h:" ./Makefile | cut -f 1 -d :$) { + $make {header}$? + } +} + +fun build_openssl_inner() +{ + let dll_ext = get_dll_ext() + let cache_root = get_cache_root() + + trust $chmod +x ./config$ + + $./config$? + make_headers()? + $make libcrypto.{dll_ext}$? + trust $cp ./libcrypto.* {cache_root}/nix-lib/$ } pub fun build_openssl() { + let source_root = get_source_root() + let cache_root = get_cache_root() + step_title("libcrypto") if pkg_exists("libcrypto"): return 0 if env_var_test("OPENSSL_LIBS") and env_var_test("OPENSSL_CFLAGS"): return 0 + + pull_source_file("openssl", "{source_root}/openssl")? + + // Using a subshell ensures we aren't cd elsewhere on failure + $(cd {source_root}/openssl && {nameof build_openssl_inner})$? + + trust env_var_set("OPENSSL_LIBS", "{cache_root}/nix-lib") + trust env_var_set("OPENSSL_CFLAGS", "-I{source_root}/openssl/include") + trust $export OPENSSL_LIBS OPENSSL_CFLAGS$ +} + +// Quick mark-as-used workaround +if false { + exit 300 + trust build_openssl_inner() } diff --git a/src/resources.ab b/src/resources.ab index 22927e3..70781b0 100644 --- a/src/resources.ab +++ b/src/resources.ab @@ -27,8 +27,10 @@ pub fun pull_source_file(member: Text, dest: Text): Null? fail 1 } + $gzip -d -c {tmpf} | tar -x -C {tmpd}$? + trust $rm -f {tmpf}$ - where = tmpd + where = "{tmpd}/{member}" } trust mv where dest diff --git a/src/term.ab b/src/term.ab index a36ec6f..dca184b 100644 --- a/src/term.ab +++ b/src/term.ab @@ -2,7 +2,7 @@ // Licensed under GNU GPLv2 // Utilities to manipulate Xterm features -import { env_var_get } from "std/env" +import { env_var_get, env_var_test } from "std/env" import { starts_with } from "std/text" let tsl = trust $tput tsl$ @@ -61,7 +61,7 @@ pub fun teardown(failure: Bool = false): Null { if failure { echo "Press any key to continue..." - trust $read -k 1 -s$ + trust $read -n 1$ } set_title() exit_alt_buffer() From d81fdb570d58d321988c3e198732c0875b4f142b Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Mon, 24 Feb 2025 11:13:24 +0100 Subject: [PATCH 18/22] Added unit entry points to individual builders Also removes the need for the "ensure used" marker --- .gitignore | 2 +- flake.lock | 14 +++++++------- src/builders/autoconf.ab | 18 ++++++++++++++++++ src/builders/boost.ab | 8 ++++---- src/builders/lowdown.ab | 8 ++++---- src/builders/nix.ab | 11 +++++++---- src/builders/nlohmann_json.ab | 6 ++++++ src/builders/openssl.ab | 9 +++++---- src/resources.ab | 29 ++++++++++++++++++++++++----- 9 files changed, 76 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index d591798..dfe7d36 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ __pycache__ result nixie/output/nix-wrapped.sh.in src/docs -src/*.sh +src/**/*.sh diff --git a/flake.lock b/flake.lock index db0cb55..bfa14f8 100644 --- a/flake.lock +++ b/flake.lock @@ -8,11 +8,11 @@ "utils": "utils" }, "locked": { - "lastModified": 1739091767, - "narHash": "sha256-6z1+TMi9lUdIWGz/RDyhCR69lCrl/tyJ0Ezfp2c18aI=", + "lastModified": 1740394272, + "narHash": "sha256-s/ZxG/ZB1hDhH2pTlHV0ANgQid18gnva6NnF/7G8nr0=", "owner": "thesola10", "repo": "amber", - "rev": "7ba5ae7b20f0b08b504c2b9e8b96b0094ef39485", + "rev": "d9acb8aa6a49ae77041fccc61c93e60f5aa429a6", "type": "github" }, "original": { @@ -178,16 +178,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1739397811, - "narHash": "sha256-xal/3k6gtnkaZH5VWsTexaeGa8092HoPoty+jj2QFEA=", + "lastModified": 1722415718, + "narHash": "sha256-5US0/pgxbMksF92k1+eOa8arJTJiPvsdZj9Dl+vJkM4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "efc60b3e477818e3b5fcec5f3a7c0aea4865ae87", + "rev": "c3392ad349a5227f4a3464dce87bcc5046692fce", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-24.11", + "ref": "nixpkgs-unstable", "repo": "nixpkgs", "type": "github" } diff --git a/src/builders/autoconf.ab b/src/builders/autoconf.ab index a30a45c..b0bf0db 100644 --- a/src/builders/autoconf.ab +++ b/src/builders/autoconf.ab @@ -40,3 +40,21 @@ pub fun build_autoconf_dep(lib_name: Text, var_name: Text, trust env_var_set("{var_name}_CFLAGS", "-I{my_source}/{inc_prefix}/include") trust $export {var_name}_LIBS {var_name}_CFLAGS$ } + +main(cmdl) +{ + if len(cmdl) < 5 { + echo "Usage: ./autoconf.sh " + echo "" + echo "See builders/autoconf.ab and builders.ab for more info" + exit 1 + } + + let lib_name = cmdl[1] + let var_name = cmdl[2] + let lib_prefix = cmdl[3] + let inc_prefix = cmdl[4] + + trust env_var_set("_NIXIE_TESTING_SKIP_TARBALL", "1") + build_autoconf_dep(lib_name, var_name, lib_prefix, inc_prefix)? +} diff --git a/src/builders/boost.ab b/src/builders/boost.ab index 6fe31d6..16665f2 100644 --- a/src/builders/boost.ab +++ b/src/builders/boost.ab @@ -60,8 +60,8 @@ pub fun build_boost() trust $export BOOST_ROOT$ } -// Quick mark-as-used workaround -if false { - exit 300 - trust build_boost_inner() +main(cmdl) +{ + trust env_var_set("_NIXIE_TESTING_SKIP_TARBALL", "1") + build_boost()? } diff --git a/src/builders/lowdown.ab b/src/builders/lowdown.ab index 7e957b4..cffc394 100644 --- a/src/builders/lowdown.ab +++ b/src/builders/lowdown.ab @@ -41,8 +41,8 @@ pub fun build_lowdown() trust env_var_set("LOWDOWN_CFLAGS", "-I{source_root}/lowdown") } -// Quick mark-as-used workaround -if false { - exit 300 - trust build_lowdown_inner() +main(cmdl) +{ + trust env_var_set("_NIXIE_TESTING_SKIP_TARBALL", "1") + build_lowdown()? } diff --git a/src/builders/nix.ab b/src/builders/nix.ab index 665f30b..f1f9948 100644 --- a/src/builders/nix.ab +++ b/src/builders/nix.ab @@ -47,9 +47,12 @@ pub fun build_nix() trust mv "{source_root}/nix/src/nix/nix" "{cache_root}/nix-static" } +main(cmdl) +{ + echo "This builder only makes sense in the wrapper context," + echo "where all dependencies were built and/or resolved." + echo "" + echo "Use ./nix --nixie-no-precompiled to test the full build chain." -// Quick mark-as-used workaround -if false { - exit 300 - trust build_nix_inner() + exit 1 } diff --git a/src/builders/nlohmann_json.ab b/src/builders/nlohmann_json.ab index 1a337b6..6789161 100644 --- a/src/builders/nlohmann_json.ab +++ b/src/builders/nlohmann_json.ab @@ -23,3 +23,9 @@ pub fun build_nlohmann_json() trust env_var_set("NLOHMANN_JSON_CFLAGS", "{source_root}/nlohmann_json/single_include") trust $export NLOHMANN_JSON_LIBS NLOHMANN_JSON_CFLAGS$ } + +main(cmdl) +{ + trust env_var_set("_NIXIE_TESTING_SKIP_TARBALL", "1") + build_nlohmann_json()? +} diff --git a/src/builders/openssl.ab b/src/builders/openssl.ab index b9c5deb..abee2ac 100644 --- a/src/builders/openssl.ab +++ b/src/builders/openssl.ab @@ -57,8 +57,9 @@ pub fun build_openssl() trust $export OPENSSL_LIBS OPENSSL_CFLAGS$ } -// Quick mark-as-used workaround -if false { - exit 300 - trust build_openssl_inner() +main(cmdl) +{ + trust env_var_set("_NIXIE_TESTING_SKIP_TARBALL", "1") + build_openssl()? } + diff --git a/src/resources.ab b/src/resources.ab index 70781b0..0b3aa66 100644 --- a/src/resources.ab +++ b/src/resources.ab @@ -3,7 +3,8 @@ // Resource retrieval from tarball or Cachix import { file_download } from "std/http" -import { env_var_get } from "std/env" +import { file_exists } from "std/fs" +import { env_var_get, env_var_test } from "std/env" import { untar } from "./common.ab" @@ -18,14 +19,32 @@ pub fun pull_source_file(member: Text, dest: Text): Null? { let SOURCE_DERIVATION = trust env_var_get("SOURCE_DERIVATION") - let where = trust untar("sources/{member}") - if status != 0 { + let where = "" + let my_status = 1 + if not env_var_test("_NIXIE_TESTING_SKIP_TARBALL") { + where = trust untar("sources/{member}") + my_status = status + } + + if env_var_test("_NIXIE_TESTING_SOURCES_DIR") and my_status != 0 { + let srcdir = trust env_var_get("_NIXIE_TESTING_SOURCES_DIR") + let tmpd = trust $mktemp -t -d nixie_{member}_XXXXXXXX$ + + if file_exists("{srcdir}/{member}.tar.gz") { + trust $gzip -d -c {srcdir}/{member}.tar.gz | tar -x -C {tmpd}$ + my_status = status + where = "{tmpd}/{member}" + } else { + my_status = 1 + } + } + + if my_status != 0 { let tmpf = trust $mktemp -t nixie_src_XXXXXXXX.tgz$ let tmpd = trust $mktemp -t -d nixie_{member}_XXXXXXXX$ - if not file_download(cachix_url(SOURCE_DERIVATION, "{member}.tar.gz"), tmpf) { + if not file_download(cachix_url(SOURCE_DERIVATION, "{member}.tar.gz"), tmpf): fail 1 - } $gzip -d -c {tmpf} | tar -x -C {tmpd}$? From 90fb7aef81c04836621e277bbfd49e102d742259 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Mon, 24 Feb 2025 13:09:39 +0100 Subject: [PATCH 19/22] Documentation + fixed Brotli configure script --- sources/00-brotli-add-automake.patch | 18 +++++++----------- sources/brotli-gen-sources-list.sh | 10 ++++++++++ sources/default.nix | 5 ++++- src/builders/autoconf.ab | 1 + src/builders/boost.ab | 5 +++++ src/builders/common.ab | 2 ++ src/builders/lowdown.ab | 6 ++++++ src/builders/nix.ab | 8 +++++++- src/builders/nlohmann_json.ab | 4 ++++ src/builders/openssl.ab | 5 +++++ src/common.ab | 2 +- src/resources.ab | 21 +++++++++++++++++++++ 12 files changed, 73 insertions(+), 14 deletions(-) create mode 100644 sources/brotli-gen-sources-list.sh diff --git a/sources/00-brotli-add-automake.patch b/sources/00-brotli-add-automake.patch index 40bb6e6..531c6ca 100644 --- a/sources/00-brotli-add-automake.patch +++ b/sources/00-brotli-add-automake.patch @@ -108,7 +108,7 @@ new file mode 100755 index 0000000..d4325b2 --- /dev/null +++ b/bootstrap -@@ -0,0 +1,36 @@ +@@ -0,0 +1,32 @@ +#!/bin/sh -e + +REQUIRED='is required, but not installed.' @@ -127,18 +127,14 @@ index 0000000..d4325b2 +mkdir m4 2>/dev/null +fi + -+BROTLI_ABI_HEX=`sed -n 's/#define BROTLI_ABI_VERSION 0x//p' c/common/version.h` -+BROTLI_ABI_INT=`echo "ibase=16;$BROTLI_ABI_HEX" | bc` -+BROTLI_ABI_CURRENT=`echo "scale=0;$BROTLI_ABI_INT / 16777216" | bc` -+BROTLI_ABI_REVISION=`echo "scale=0;$BROTLI_ABI_INT / 4096 % 4096" | bc` -+BROTLI_ABI_AGE=`echo "scale=0;$BROTLI_ABI_INT % 4096" | bc` ++BROTLI_ABI_CURRENT=`sed -n 's/#define BROTLI_ABI_CURRENT //p' c/common/version.h` ++BROTLI_ABI_REVISION=`sed -n 's/#define BROTLI_ABI_REVISION //p' c/common/version.h` ++BROTLI_ABI_AGE=`sed -n 's/#define BROTLI_ABI_AGE //p' c/common/version.h` +BROTLI_ABI_INFO="$BROTLI_ABI_CURRENT:$BROTLI_ABI_REVISION:$BROTLI_ABI_AGE" + -+BROTLI_VERSION_HEX=`sed -n 's/#define BROTLI_VERSION 0x//p' c/common/version.h` -+BROTLI_VERSION_INT=`echo "ibase=16;$BROTLI_VERSION_HEX" | bc` -+BROTLI_VERSION_MAJOR=`echo "scale=0;$BROTLI_VERSION_INT / 16777216" | bc` -+BROTLI_VERSION_MINOR=`echo "scale=0;$BROTLI_VERSION_INT / 4096 % 4096" | bc` -+BROTLI_VERSION_PATCH=`echo "scale=0;$BROTLI_VERSION_INT % 4096" | bc` ++BROTLI_VERSION_MAJOR=`sed -n 's/#define BROTLI_VERSION_MAJOR //p' c/common/version.h` ++BROTLI_VERSION_MINOR=`sed -n 's/#define BROTLI_VERSION_MINOR //p' c/common/version.h` ++BROTLI_VERSION_PATCH=`sed -n 's/#define BROTLI_VERSION_PATCH //p' c/common/version.h` +BROTLI_VERSION="$BROTLI_VERSION_MAJOR.$BROTLI_VERSION_MINOR.$BROTLI_VERSION_PATCH" + +sed -i.bak "$SED_ERE" "s/[0-9]+:[0-9]+:[0-9]+/$BROTLI_ABI_INFO/" Makefile.am diff --git a/sources/brotli-gen-sources-list.sh b/sources/brotli-gen-sources-list.sh new file mode 100644 index 0000000..6da3123 --- /dev/null +++ b/sources/brotli-gen-sources-list.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +echo "BROTLI_CLI_C = " c/tools/*.c +echo "BROTLI_COMMON_C = " c/common/*.c +echo "BROTLI_COMMON_H = " c/common/*.h +echo "BROTLI_DEC_C = " c/dec/*.c +echo "BROTLI_DEC_H = " c/dec/*.h +echo "BROTLI_ENC_C = " c/enc/*.c +echo "BROTLI_ENC_H = " c/enc/*.h +echo "BROTLI_INCLUDE = " c/include/brotli/*.h diff --git a/sources/default.nix b/sources/default.nix index 747cc13..62133df 100644 --- a/sources/default.nix +++ b/sources/default.nix @@ -47,7 +47,10 @@ let brotli_configured_src = mkConfiguredSrc { pkg = brotli; patches = [ ./00-brotli-add-automake.patch ]; - confScript = "true"; + confScript = '' + sh ${./brotli-gen-sources-list.sh} > ./scripts/sources.lst + ./bootstrap + ''; dest = "libbrotlicommon"; }; diff --git a/src/builders/autoconf.ab b/src/builders/autoconf.ab index b0bf0db..99f6a88 100644 --- a/src/builders/autoconf.ab +++ b/src/builders/autoconf.ab @@ -56,5 +56,6 @@ main(cmdl) let inc_prefix = cmdl[4] trust env_var_set("_NIXIE_TESTING_SKIP_TARBALL", "1") + trust env_var_set("step_total", "1") build_autoconf_dep(lib_name, var_name, lib_prefix, inc_prefix)? } diff --git a/src/builders/boost.ab b/src/builders/boost.ab index 16665f2..9cbdc2a 100644 --- a/src/builders/boost.ab +++ b/src/builders/boost.ab @@ -31,6 +31,7 @@ fun find_boost_libs(libs: [Text]): Bool return true } +/// This function runs within the source directory, in a subshell. fun build_boost_inner() { let args = [ "--static" ] @@ -43,6 +44,9 @@ fun build_boost_inner() $./b2 "{args}"$? } +/// This is the core function for the Boost builder. +/// +/// It exports the `BOOST_ROOT` variable. pub fun build_boost() { let source_root = get_source_root() @@ -63,5 +67,6 @@ pub fun build_boost() main(cmdl) { trust env_var_set("_NIXIE_TESTING_SKIP_TARBALL", "1") + trust env_var_set("step_total", "1") build_boost()? } diff --git a/src/builders/common.ab b/src/builders/common.ab index 214ccbc..caeec57 100644 --- a/src/builders/common.ab +++ b/src/builders/common.ab @@ -20,6 +20,7 @@ pub fun pkg_exists(package: Text): Bool return true } +/// Change the terminal title to a build step, and increments the step counter. pub fun step_title(name: Text): Null { let step_total = trust env_var_get("step_total") @@ -31,6 +32,7 @@ pub fun step_title(name: Text): Null step_current += 1 } +/// Obtain the base directory where source packages are extracted. pub fun get_source_root(): Text { let repo_root = get_repo_root() diff --git a/src/builders/lowdown.ab b/src/builders/lowdown.ab index cffc394..d16a65d 100644 --- a/src/builders/lowdown.ab +++ b/src/builders/lowdown.ab @@ -9,11 +9,13 @@ import { get_osname, get_cache_root } from "../platform.ab" import { pkg_exists, step_title, get_source_root } from "./common.ab" +/// Workaround for the default Makefile not building a shared library on macOS. fun macos_build_post() { $cc -shared -o liblowdown.1.dylib *.o$? } +/// This function runs within the source directory, in a subshell. fun build_lowdown_inner() { $./configure$? @@ -23,6 +25,9 @@ fun build_lowdown_inner() macos_build_post()? } +/// This is the core function for the Lowdown builder. +/// +/// It exports the `LOWDOWN_LIBS` and `LOWDOWN_CFLAGS` variables. pub fun build_lowdown() { let source_root = get_source_root() @@ -44,5 +49,6 @@ pub fun build_lowdown() main(cmdl) { trust env_var_set("_NIXIE_TESTING_SKIP_TARBALL", "1") + trust env_var_set("step_total", "1") build_lowdown()? } diff --git a/src/builders/nix.ab b/src/builders/nix.ab index f1f9948..533e8bf 100644 --- a/src/builders/nix.ab +++ b/src/builders/nix.ab @@ -9,6 +9,7 @@ import { get_cache_root } from "../platform.ab" import { step_title, get_source_root } from "./common.ab" +/// This function runs within the source directory, in a subshell. fun build_nix_inner() { let source_root = get_source_root() @@ -22,11 +23,16 @@ fun build_nix_inner() -Dlibutil:cpuid=disabled \ -Ddoc-gen=false \ -Dunit-tests=false \ - -Dbindings=false$? + -Dbindings=false \ + ..$? ${venv}/bin/ninja$? } +/// This is the core function for the final Nix builder. +/// +/// It produces a Nix executable in the `{cache_root}/nix-static` location, +/// where the Nix runner expects it. pub fun build_nix() { let source_root = get_source_root() diff --git a/src/builders/nlohmann_json.ab b/src/builders/nlohmann_json.ab index 6789161..18209b8 100644 --- a/src/builders/nlohmann_json.ab +++ b/src/builders/nlohmann_json.ab @@ -8,6 +8,9 @@ import { pull_source_file } from "../resources.ab" import { pkg_exists, step_title, get_source_root } from "./common.ab" +/// This is the core function for the NLohmann JSON library. +/// +/// It exports the `LNLOHMANN_JSON_LIBS` and `NLOHMANN_JSON_CFLAGS` variables. pub fun build_nlohmann_json() { let source_root = get_source_root() @@ -27,5 +30,6 @@ pub fun build_nlohmann_json() main(cmdl) { trust env_var_set("_NIXIE_TESTING_SKIP_TARBALL", "1") + trust env_var_set("step_total", "1") build_nlohmann_json()? } diff --git a/src/builders/openssl.ab b/src/builders/openssl.ab index abee2ac..fccdf47 100644 --- a/src/builders/openssl.ab +++ b/src/builders/openssl.ab @@ -21,6 +21,7 @@ fun make_headers() } } +/// This function runs within the source directory, in a subshell. fun build_openssl_inner() { let dll_ext = get_dll_ext() @@ -35,6 +36,9 @@ fun build_openssl_inner() trust $cp ./libcrypto.* {cache_root}/nix-lib/$ } +/// This is the core function for the OpenSSL builder. +/// +/// It exports the `OPENSSL_LIBS` and `OPENSSL_CFLAGS` variables. pub fun build_openssl() { let source_root = get_source_root() @@ -60,6 +64,7 @@ pub fun build_openssl() main(cmdl) { trust env_var_set("_NIXIE_TESTING_SKIP_TARBALL", "1") + trust env_var_set("step_total", "1") build_openssl()? } diff --git a/src/common.ab b/src/common.ab index ab672d0..2a9f7f4 100644 --- a/src/common.ab +++ b/src/common.ab @@ -19,7 +19,7 @@ if text_contains(me, "/") { SELF = "{PWD}/{me}" } - // This second pass check if the result of readlink is relative + // This second pass checks if the result of readlink is relative if { rl == "": SELF = SELF not starts_with(rl, "/"): SELF = "{PWD}/{rl}" diff --git a/src/resources.ab b/src/resources.ab index 0b3aa66..5e3a6dc 100644 --- a/src/resources.ab +++ b/src/resources.ab @@ -15,12 +15,24 @@ fun cachix_url(derivation: Text, member: Text): Text return "https://{SOURCE_CACHE}/serve/{derivation}/{member}" } +/// Unpack a known source code package into the requested target directory. +/// +/// This function will try the following methods, in order: +/// - Extract from the embedded tarball +/// - Pull tarball from directory marked by `_NIXIE_TESTING_SOURCES_DIR` +/// - Download tarball from known Cachix URL +/// +/// ### Arguments: +/// - `member`: The name of the source package to unpack +/// - `dest`: The target directory to move the package contents into pub fun pull_source_file(member: Text, dest: Text): Null? { let SOURCE_DERIVATION = trust env_var_get("SOURCE_DERIVATION") let where = "" let my_status = 1 + + // This allows individual builders' entry points to not fail if not env_var_test("_NIXIE_TESTING_SKIP_TARBALL") { where = trust untar("sources/{member}") my_status = status @@ -55,6 +67,15 @@ pub fun pull_source_file(member: Text, dest: Text): Null? trust mv where dest } +/// Retrieve a known precompiled file and move it to the requested target. +/// +/// This function will try the following methods, in order: +/// - Extract from the embedded tarball +/// - Download file from known Cachix URL +/// +/// ### Arguments: +/// - `member`: The filename of the file to retrieve +/// - `dest`: The target filename to move the file into pub fun pull_binary(member: Text, dest: Text): Null? { let NIX_BINS_DERIVATION = trust env_var_get("NIX_BINS_DERIVATION") From ce6b170a5428ef9c11cf9648af386f06627d69df Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Mon, 24 Feb 2025 17:59:27 +0100 Subject: [PATCH 20/22] Updated sources and fixed source build --- flake.lock | 36 +++++++++++++++++------------------ flake.nix | 4 +++- sources/Makefile | 4 ++-- sources/default.nix | 13 ++++++++----- src/builders.ab | 8 ++++---- src/builders/autoconf.ab | 6 +++++- src/builders/boost.ab | 4 ++-- src/builders/lowdown.ab | 3 +++ src/builders/nix.ab | 2 ++ src/builders/nlohmann_json.ab | 3 +++ src/builders/openssl.ab | 3 +++ 11 files changed, 53 insertions(+), 33 deletions(-) diff --git a/flake.lock b/flake.lock index bfa14f8..07c0328 100644 --- a/flake.lock +++ b/flake.lock @@ -48,11 +48,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1733328505, + "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", "type": "github" }, "original": { @@ -69,11 +69,11 @@ ] }, "locked": { - "lastModified": 1730504689, - "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=", + "lastModified": 1733312601, + "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "506278e768c2a08bec68eb62932193e341f55c90", + "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", "type": "github" }, "original": { @@ -87,11 +87,11 @@ "systems": "systems_2" }, "locked": { - "lastModified": 1694529238, - "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -117,11 +117,11 @@ ] }, "locked": { - "lastModified": 1732021966, - "narHash": "sha256-mnTbjpdqF0luOkou8ZFi2asa1N3AA2CchR/RqCNmsGE=", + "lastModified": 1734279981, + "narHash": "sha256-NdaCraHPp8iYMWzdXAt5Nv6sA3MUzlCiGiR586TCwo0=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "3308484d1a443fc5bc92012435d79e80458fe43c", + "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785", "type": "github" }, "original": { @@ -226,11 +226,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1739397811, - "narHash": "sha256-xal/3k6gtnkaZH5VWsTexaeGa8092HoPoty+jj2QFEA=", + "lastModified": 1734359947, + "narHash": "sha256-1Noao/H+N8nFB4Beoy8fgwrcOQLVm9o4zKW1ODaqK9E=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "efc60b3e477818e3b5fcec5f3a7c0aea4865ae87", + "rev": "48d12d5e70ee91fe8481378e540433a7303dbf6a", "type": "github" }, "original": { @@ -242,11 +242,11 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1697484591, - "narHash": "sha256-ahV2m9CeTLkTOo0hZNf3miMKTOOHtWKIBnjxNeFdx5A=", + "lastModified": 1740402658, + "narHash": "sha256-r9gIbKRxH+tzXkXfgYoGVeywS9r3zirUya2ibrrqNeE=", "owner": "nixos", "repo": "nixpkgs", - "rev": "997dcad8d0bc3e80adfec3d0f2a808e8945a4547", + "rev": "9de281f1057097fe911e69fc5816268b020f61af", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 065ddbb..fc54178 100644 --- a/flake.nix +++ b/flake.nix @@ -21,7 +21,9 @@ { packages = rec { default = nixie; nixie = pkgs.callPackage ./. { inherit amber-lang; }; - sources = pkgs.callPackage ./sources {}; + sources = pkgs.callPackage ./sources + { nix-source = nix; + }; static-bins = import ./static-bins { inherit nixpkgs fakedir pkgs; nix-source = nix; diff --git a/sources/Makefile b/sources/Makefile index b7ce1f5..1eda23d 100644 --- a/sources/Makefile +++ b/sources/Makefile @@ -10,8 +10,8 @@ WGET := wget BOOST_VER := 1.87.0 BOOST_ARCHIVE := $(WORKDIR)/boost.tar.bz2 -BOOST_ADD_HEADERS = core,utility,io,system,thread,context,lexical_cast,config,format,coroutine2,container,chrono,atomic,predef,move,assert,detail,type_traits,intrusive,mpl,date_time,bind,align,preprocessor,ratio,exception,smart_ptr,numeric,functional,container_hash,describe,tuple,iterator,function,integer,type_index,algorithm,range,concept,optional,mp11 -BOOST_ADD_MODULES = system,thread,context,format,coroutine2,container,chrono,atomic,optional +BOOST_ADD_HEADERS = core,utility,io,system,thread,context,lexical_cast,config,format,coroutine,container,chrono,atomic,predef,move,assert,detail,type_traits,intrusive,mpl,date_time,bind,align,preprocessor,ratio,exception,smart_ptr,numeric,functional,container_hash,describe,tuple,iterator,function,integer,type_index,algorithm,range,concept,optional,mp11 +BOOST_ADD_MODULES = assert,static_assert,throw_exception,integer,type_traits,move,mpl,ratio,variant2,mp11,winapi,typeof,utility,intrusive,pool,smart_ptr,exception,system,predef,thread,context,format,coroutine,container,chrono,atomic,optional boost-shaved.tar.gz: $(BOOST_ARCHIVE) @mkdir -p $(WORKDIR) diff --git a/sources/default.nix b/sources/default.nix index 62133df..7f0c976 100644 --- a/sources/default.nix +++ b/sources/default.nix @@ -1,12 +1,15 @@ { stdenv, boost, openssl, lowdown, nlohmann_json, brotli, libsodium, editline , gnutar, coreutils, findutils, python3, nix -, meson, automake, autoconf-archive, autoconf, m4, bc, libtool, pkg-config, ... }: +, automake, autoconf-archive, autoconf, m4, bc, libtool, pkg-config +# External source for Nix +, nix-source ? nix.src +, ... }: let - mkConfiguredSrc = { pkg, confScript, patches ? [], dest?pkg.pname }: + mkConfiguredSrc = { pkg, confScript, src ? pkg.src, patches ? pkg.patches, dest ? pkg.pname }: stdenv.mkDerivation { - inherit (pkg) version src; - inherit dest patches; + inherit (pkg) version; + inherit dest patches src; pname = "${pkg.pname}-configured-sources"; configurePhase = confScript; @@ -19,7 +22,6 @@ let bc libtool pkg-config - meson ]; dontBuild = true; @@ -33,6 +35,7 @@ let nix_configured_src = mkConfiguredSrc { pkg = nix; + src = nix-source; confScript = '' mkdir -p $out cp -r . $out/nix diff --git a/src/builders.ab b/src/builders.ab index b771ab4..d3ef21c 100644 --- a/src/builders.ab +++ b/src/builders.ab @@ -67,9 +67,9 @@ pub fun try_build_nix() build_boost()? build_nlohmann_json()? build_lowdown()? - // pkgconf name env var lib prefix include prefix - build_autoconf_dep("libbrotlicommon", "LIBBROTLI", "", "c")? - build_autoconf_dep("libsodium", "SODIUM", "src/libsodium", "src/libsodium")? - build_autoconf_dep("libeditline", "EDITLINE", "src")? + // pkgconf name env var lib prefix include prefix + build_autoconf_dep("libbrotlicommon", "LIBBROTLI", "", "c/include")? + build_autoconf_dep("libsodium", "SODIUM", "src/libsodium", "src/libsodium/include")? + build_autoconf_dep("libeditline", "EDITLINE", "src", "include")? build_nix()? } diff --git a/src/builders/autoconf.ab b/src/builders/autoconf.ab index 99f6a88..bcf765f 100644 --- a/src/builders/autoconf.ab +++ b/src/builders/autoconf.ab @@ -32,13 +32,17 @@ pub fun build_autoconf_dep(lib_name: Text, var_name: Text, pull_source_file(lib_name, my_source)? - $(cd {my_source} && ./configure && make)$? + $(unset C_INCLUDE_PATH CPLUS_INCLUDE_PATH && cd {my_source} && ./configure && make)$? $cp {my_source}/{lib_prefix}/.libs/* {cache_root}/nix-lib/$? + trust $rm {cache_root}/nix-lib/*.o$ trust env_var_set("{var_name}_LIBS", "{cache_root}/nix-lib") trust env_var_set("{var_name}_CFLAGS", "-I{my_source}/{inc_prefix}/include") trust $export {var_name}_LIBS {var_name}_CFLAGS$ + + trust $export C_INCLUDE_PATH={my_source}/{inc_prefix}:\$C_INCLUDE_PATH$ + trust $export CPLUS_INCLUDE_PATH={my_source}/{inc_prefix}:\$CPLUS_INCLUDE_PATH$ } main(cmdl) diff --git a/src/builders/boost.ab b/src/builders/boost.ab index 9cbdc2a..da268bd 100644 --- a/src/builders/boost.ab +++ b/src/builders/boost.ab @@ -9,7 +9,7 @@ import { pull_source_file } from "../resources.ab" import { pkg_exists, step_title, get_source_root } from "./common.ab" -const modules = ["chrono", "container", "context", "system", "thread"] +const modules = ["predef", "chrono", "container", "context", "coroutine", "system", "thread"] /// Check that the Boost modules we need exist on the system. /// @@ -34,7 +34,7 @@ fun find_boost_libs(libs: [Text]): Bool /// This function runs within the source directory, in a subshell. fun build_boost_inner() { - let args = [ "--static" ] + let args = [ "variant=release", "link=static", "--stagedir=." ] for mod in modules { args += [ "--with-{mod}" ] diff --git a/src/builders/lowdown.ab b/src/builders/lowdown.ab index d16a65d..5902aac 100644 --- a/src/builders/lowdown.ab +++ b/src/builders/lowdown.ab @@ -44,6 +44,9 @@ pub fun build_lowdown() trust env_var_set("LOWDOWN_LIBS", "{cache_root}/nix-lib") trust env_var_set("LOWDOWN_CFLAGS", "-I{source_root}/lowdown") + + trust $export C_INCLUDE_PATH={source_root}/lowdown:\$C_INCLUDE_PATH$ + trust $export CPLUS_INCLUDE_PATH={source_root}/lowdown:\$CPLUS_INCLUDE_PATH$ } main(cmdl) diff --git a/src/builders/nix.ab b/src/builders/nix.ab index 533e8bf..f3709aa 100644 --- a/src/builders/nix.ab +++ b/src/builders/nix.ab @@ -46,6 +46,8 @@ pub fun build_nix() $python3 -m venv --system-site-packages "{venv}"$? + trust $export LIBRARY_PATH={cache_root}/nix-lib:\$LIBRARY_PATH$ + ${venv}/bin/pip install meson ninja$? $(cd {source_root}/nix && {nameof build_nix_inner})$? diff --git a/src/builders/nlohmann_json.ab b/src/builders/nlohmann_json.ab index 18209b8..1757f94 100644 --- a/src/builders/nlohmann_json.ab +++ b/src/builders/nlohmann_json.ab @@ -25,6 +25,9 @@ pub fun build_nlohmann_json() trust env_var_set("NLOHMANN_JSON_LIBS", "{source_root}/nlohmann_json/single_include") trust env_var_set("NLOHMANN_JSON_CFLAGS", "{source_root}/nlohmann_json/single_include") trust $export NLOHMANN_JSON_LIBS NLOHMANN_JSON_CFLAGS$ + + trust $export C_INCLUDE_PATH={source_root}/nlohmann_json/single_include:\$C_INCLUDE_PATH$ + trust $export CPLUS_INCLUDE_PATH={source_root}/nlohmann_json/single_include:\$CPLUS_INCLUDE_PATH$ } main(cmdl) diff --git a/src/builders/openssl.ab b/src/builders/openssl.ab index fccdf47..ae4fce2 100644 --- a/src/builders/openssl.ab +++ b/src/builders/openssl.ab @@ -59,6 +59,9 @@ pub fun build_openssl() trust env_var_set("OPENSSL_LIBS", "{cache_root}/nix-lib") trust env_var_set("OPENSSL_CFLAGS", "-I{source_root}/openssl/include") trust $export OPENSSL_LIBS OPENSSL_CFLAGS$ + + trust $export C_INCLUDE_PATH={source_root}/openssl/include:\$C_INCLUDE_PATH$ + trust $export CPLUS_INCLUDE_PATH={source_root}/openssl/include:\$CPLUS_INCLUDE_PATH$ } main(cmdl) From 15db1e2be5c1fc0b759d7f9ca0ea7506e8082a20 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Tue, 25 Feb 2025 11:20:52 +0100 Subject: [PATCH 21/22] Source build now relies on pkg-config --- sources/default.nix | 7 ++++++- src/builders.ab | 19 +++++++++++-------- src/builders/autoconf.ab | 29 ++++++++--------------------- src/builders/lowdown.ab | 11 ++++------- src/builders/nix.ab | 13 ++++++------- src/builders/nlohmann_json.ab | 16 +++++++++------- src/builders/openssl.ab | 20 ++++++++++---------- src/cli.ab | 2 +- src/nix.ab | 8 ++++---- src/resources.ab | 3 +++ 10 files changed, 62 insertions(+), 66 deletions(-) diff --git a/sources/default.nix b/sources/default.nix index 7f0c976..a3f443a 100644 --- a/sources/default.nix +++ b/sources/default.nix @@ -1,5 +1,5 @@ { stdenv, boost, openssl, lowdown, nlohmann_json, brotli, libsodium, editline -, gnutar, coreutils, findutils, python3, nix +, gnutar, coreutils, findutils, python3, nix, libarchive , automake, autoconf-archive, autoconf, m4, bc, libtool, pkg-config # External source for Nix , nix-source ? nix.src @@ -56,6 +56,10 @@ let ''; dest = "libbrotlicommon"; }; + libarchive_configured_src = mkConfiguredSrc + { pkg = libarchive; + confScript = "./build/autogen.sh"; + }; srcs_simple = [ openssl @@ -69,6 +73,7 @@ let [ nix_configured_src editline_configured_src brotli_configured_src + libarchive_configured_src ]; in stdenv.mkDerivation { name = "nixie-sources"; diff --git a/src/builders.ab b/src/builders.ab index d3ef21c..425deee 100644 --- a/src/builders.ab +++ b/src/builders.ab @@ -20,6 +20,8 @@ import { build_autoconf_dep } from "./builders/autoconf.ab" import { build_nix } from "./builders/nix.ab" /// Export libraries found inside the macOS SDK for building Nix +/// +/// TODO: is this still required with Meson? fun darwin_export_sdk() { // Calling xcrun should prompt the user to install the macOS SDK. @@ -46,8 +48,8 @@ fun darwin_export_sdk() /// Build Nix and its dependencies locally, then place it in the expected location /// in the user's cache directory. /// -/// This process **requires**, among other things, `pkg-config` to determine -/// which dependencies are already met and avoid building them. +/// This process **requires**, among other things, `pkg-config` due to it being +/// the only detection method for many dependencies in Meson. pub fun try_build_nix() { let cache_root = get_cache_root() @@ -58,18 +60,19 @@ pub fun try_build_nix() if get_osname() == "Darwin": darwin_export_sdk() - trust env_var_set("step_total", "8") + trust env_var_set("step_total", "9") dir_create(get_source_root()) - dir_create("{cache_root}/nix-lib") + dir_create("{cache_root}/nix-deps/lib/pkgconfig") build_openssl()? build_boost()? build_nlohmann_json()? build_lowdown()? - // pkgconf name env var lib prefix include prefix - build_autoconf_dep("libbrotlicommon", "LIBBROTLI", "", "c/include")? - build_autoconf_dep("libsodium", "SODIUM", "src/libsodium", "src/libsodium/include")? - build_autoconf_dep("libeditline", "EDITLINE", "src", "include")? + // pkgconf name include prefix + build_autoconf_dep("libbrotlicommon", "c/include")? + build_autoconf_dep("libsodium", "src/libsodium/include")? + build_autoconf_dep("libeditline", "include")? + build_autoconf_dep("libarchive", "libarchive")? build_nix()? } diff --git a/src/builders/autoconf.ab b/src/builders/autoconf.ab index bcf765f..b0a1e2d 100644 --- a/src/builders/autoconf.ab +++ b/src/builders/autoconf.ab @@ -13,12 +13,8 @@ import { pkg_exists, step_title, get_source_root } from "./common.ab" /// /// ### Arguments: /// - `lib_name`: The library to check for with `pkg-config` -/// - `var_name`: The variable to export library locations -/// - `lib_prefix`: Where in the source tree the resulting libraries are found /// - `inc_prefix`: Where in the source tree the build headers are found -pub fun build_autoconf_dep(lib_name: Text, var_name: Text, - lib_prefix: Text = "", - inc_prefix: Text = ""): Null? +pub fun build_autoconf_dep(lib_name: Text, inc_prefix: Text = ""): Null? { let source_root = get_source_root() let cache_root = get_cache_root() @@ -32,34 +28,25 @@ pub fun build_autoconf_dep(lib_name: Text, var_name: Text, pull_source_file(lib_name, my_source)? - $(unset C_INCLUDE_PATH CPLUS_INCLUDE_PATH && cd {my_source} && ./configure && make)$? - - $cp {my_source}/{lib_prefix}/.libs/* {cache_root}/nix-lib/$? - trust $rm {cache_root}/nix-lib/*.o$ - - trust env_var_set("{var_name}_LIBS", "{cache_root}/nix-lib") - trust env_var_set("{var_name}_CFLAGS", "-I{my_source}/{inc_prefix}/include") - trust $export {var_name}_LIBS {var_name}_CFLAGS$ - - trust $export C_INCLUDE_PATH={my_source}/{inc_prefix}:\$C_INCLUDE_PATH$ - trust $export CPLUS_INCLUDE_PATH={my_source}/{inc_prefix}:\$CPLUS_INCLUDE_PATH$ + $( unset C_INCLUDE_PATH CPLUS_INCLUDE_PATH \ + && cd {my_source} \ + && ./configure --prefix={cache_root}/nix-deps \ + && make && make install )$? } main(cmdl) { if len(cmdl) < 5 { - echo "Usage: ./autoconf.sh " + echo "Usage: ./autoconf.sh " echo "" echo "See builders/autoconf.ab and builders.ab for more info" exit 1 } let lib_name = cmdl[1] - let var_name = cmdl[2] - let lib_prefix = cmdl[3] - let inc_prefix = cmdl[4] + let inc_prefix = cmdl[2] trust env_var_set("_NIXIE_TESTING_SKIP_TARBALL", "1") trust env_var_set("step_total", "1") - build_autoconf_dep(lib_name, var_name, lib_prefix, inc_prefix)? + build_autoconf_dep(lib_name, inc_prefix)? } diff --git a/src/builders/lowdown.ab b/src/builders/lowdown.ab index 5902aac..5b34812 100644 --- a/src/builders/lowdown.ab +++ b/src/builders/lowdown.ab @@ -18,11 +18,14 @@ fun macos_build_post() /// This function runs within the source directory, in a subshell. fun build_lowdown_inner() { - $./configure$? + let cache_root = get_cache_root() + + $./configure PREFIX={cache_root}/nix-deps$? $make$? if get_osname() == "Darwin": macos_build_post()? + $make install_shared$? } /// This is the core function for the Lowdown builder. @@ -41,12 +44,6 @@ pub fun build_lowdown() pull_source_file("lowdown", "{source_root}/lowdown")? $(cd {source_root}/lowdown && {nameof build_lowdown_inner})$? - - trust env_var_set("LOWDOWN_LIBS", "{cache_root}/nix-lib") - trust env_var_set("LOWDOWN_CFLAGS", "-I{source_root}/lowdown") - - trust $export C_INCLUDE_PATH={source_root}/lowdown:\$C_INCLUDE_PATH$ - trust $export CPLUS_INCLUDE_PATH={source_root}/lowdown:\$CPLUS_INCLUDE_PATH$ } main(cmdl) diff --git a/src/builders/nix.ab b/src/builders/nix.ab index f3709aa..effbe46 100644 --- a/src/builders/nix.ab +++ b/src/builders/nix.ab @@ -2,6 +2,7 @@ // Licensed under GNU GPLv2 // Builder for the Nix package manager itself +import { env_var_set } from "std/env" import { file_exists, dir_exists, dir_create } from "std/fs" import { pull_source_file } from "../resources.ab" @@ -46,7 +47,8 @@ pub fun build_nix() $python3 -m venv --system-site-packages "{venv}"$? - trust $export LIBRARY_PATH={cache_root}/nix-lib:\$LIBRARY_PATH$ + trust $export LIBRARY_PATH={cache_root}/nix-deps/lib:\$LIBRARY_PATH$ + trust $export PKG_CONFIG_PATH={cache_root}/nix-deps/lib/pkgconfig:{cache_root}/nix-deps/share/pkgconfig:\$PKG_CONFIG_PATH$ ${venv}/bin/pip install meson ninja$? @@ -57,10 +59,7 @@ pub fun build_nix() main(cmdl) { - echo "This builder only makes sense in the wrapper context," - echo "where all dependencies were built and/or resolved." - echo "" - echo "Use ./nix --nixie-no-precompiled to test the full build chain." - - exit 1 + trust env_var_set("_nixie_testing_skip_tarball", "1") + trust env_var_set("step_total", "1") + build_nix()? } diff --git a/src/builders/nlohmann_json.ab b/src/builders/nlohmann_json.ab index 1757f94..a8082ca 100644 --- a/src/builders/nlohmann_json.ab +++ b/src/builders/nlohmann_json.ab @@ -3,17 +3,18 @@ // Builder for NLohmann's JSON library import { env_var_set } from "std/env" +import { file_write } from "std/fs" import { pull_source_file } from "../resources.ab" +import { get_cache_root } from "../platform.ab" import { pkg_exists, step_title, get_source_root } from "./common.ab" /// This is the core function for the NLohmann JSON library. -/// -/// It exports the `LNLOHMANN_JSON_LIBS` and `NLOHMANN_JSON_CFLAGS` variables. pub fun build_nlohmann_json() { let source_root = get_source_root() + let cache_root = get_cache_root() step_title("nlohmann_json") @@ -22,12 +23,13 @@ pub fun build_nlohmann_json() pull_source_file("nlohmann_json", "{source_root}/nlohmann_json")? - trust env_var_set("NLOHMANN_JSON_LIBS", "{source_root}/nlohmann_json/single_include") - trust env_var_set("NLOHMANN_JSON_CFLAGS", "{source_root}/nlohmann_json/single_include") - trust $export NLOHMANN_JSON_LIBS NLOHMANN_JSON_CFLAGS$ + let version = trust $grep "^version:" {source_root}/nlohmann_json/wsjcpp.yml | cut -d '"' -f 2 | cut -d 'v' -f 2$ - trust $export C_INCLUDE_PATH={source_root}/nlohmann_json/single_include:\$C_INCLUDE_PATH$ - trust $export CPLUS_INCLUDE_PATH={source_root}/nlohmann_json/single_include:\$CPLUS_INCLUDE_PATH$ + file_write("{cache_root}/nix-deps/lib/pkgconfig/nlohmann_json.pc", +"Name: nlohmann_json +Version: {version} +Description: JSON for Modern C++ +Cflags: -I{source_root}/nlohmann_json/include")? } main(cmdl) diff --git a/src/builders/openssl.ab b/src/builders/openssl.ab index ae4fce2..409fd7e 100644 --- a/src/builders/openssl.ab +++ b/src/builders/openssl.ab @@ -31,9 +31,16 @@ fun build_openssl_inner() $./config$? make_headers()? - $make libcrypto.{dll_ext}$? - - trust $cp ./libcrypto.* {cache_root}/nix-lib/$ + $make libcrypto.{dll_ext} libcrypto.pc$? + + // Building libssl is: + // - hard + // - broken + // - a very bad idea anyway + // so we're performing the install ourselves + trust $cp ./libcrypto.* {cache_root}/nix-deps/lib/$ + trust $cp ./libcrypto.pc {cache_root}/nix-deps/lib/pkgconfig$ + trust $cp -r ./include {cache_root}/nix-deps/$ } /// This is the core function for the OpenSSL builder. @@ -55,13 +62,6 @@ pub fun build_openssl() // Using a subshell ensures we aren't cd elsewhere on failure $(cd {source_root}/openssl && {nameof build_openssl_inner})$? - - trust env_var_set("OPENSSL_LIBS", "{cache_root}/nix-lib") - trust env_var_set("OPENSSL_CFLAGS", "-I{source_root}/openssl/include") - trust $export OPENSSL_LIBS OPENSSL_CFLAGS$ - - trust $export C_INCLUDE_PATH={source_root}/openssl/include:\$C_INCLUDE_PATH$ - trust $export CPLUS_INCLUDE_PATH={source_root}/openssl/include:\$CPLUS_INCLUDE_PATH$ } main(cmdl) diff --git a/src/cli.ab b/src/cli.ab index 9dcc915..a66bbd3 100644 --- a/src/cli.ab +++ b/src/cli.ab @@ -56,7 +56,7 @@ fun cmd_cleanup() trust $rm -rf {nix_root}$ echo "Removing retrieved Nix binaries..." - trust $rm -rf {cache_root}/nix-static {cache_root}/nix-lib$ + trust $rm -rf {cache_root}/nix-static {cache_root}/nix-lib {cache_root}/nix-deps$ exit 0 } diff --git a/src/nix.ab b/src/nix.ab index 78d6261..aa566aa 100644 --- a/src/nix.ab +++ b/src/nix.ab @@ -46,7 +46,7 @@ fun get_nix() let system = get_system() let nix_path = "{cache_root}/nix-static" - let fakedir_path = "{cache_root}/nix-lib/libfakedir.dylib" + let fakedir_path = "{cache_root}/nix-deps/lib/libfakedir.dylib" enter_alt_buffer() set_title("Building Nix...") @@ -56,7 +56,7 @@ fun get_nix() // Unpack fakedir as needed if osname == "Darwin" and not file_exists(fakedir_path) { - dir_create("{cache_root}/nix-lib") + dir_create("{cache_root}/nix-deps/lib") pull_binary("libfakedir.dylib", fakedir_path) failed { teardown(true) fail 1 @@ -176,7 +176,7 @@ fun launch_darwin_workaround(name: Text, nix_path: Text, args: [Text]): Null let cache_root = get_cache_root() let nix_root = get_nix_root() - let fakedir_path = "{cache_root}/nix-lib/libfakedir.dylib" + let fakedir_path = "{cache_root}/nix-deps/lib/libfakedir.dylib" trust env_var_set("FAKEDIR_PATTERN", "/nix") trust env_var_set("FAKEDIR_TARGET", "{nix_root}/nix") @@ -187,7 +187,7 @@ fun launch_darwin_workaround(name: Text, nix_path: Text, args: [Text]): Null // Unfortunately, this requires us to disable the Nix sandbox entirely. trust $ _NIX_TEST_NO_SANDBOX=1 \ DYLD_INSERT_LIBRARIES="{fakedir_path}" \ - DYLD_LIBRARY_PATH="{cache_root}/nix-lib" \ + DYLD_LIBRARY_PATH="{cache_root}/nix-deps/lib" \ exec -a {name} {nix_path} "{args}"$ } diff --git a/src/resources.ab b/src/resources.ab index 5e3a6dc..65cc3b1 100644 --- a/src/resources.ab +++ b/src/resources.ab @@ -64,6 +64,9 @@ pub fun pull_source_file(member: Text, dest: Text): Null? where = "{tmpd}/{member}" } + // Our build method means source dirs accumulate state, we're better off + // throwing them away. + trust $rm -rf {dest}$ trust mv where dest } From 7867426b0e3764b4de44886a4e62a82adf6a3824 Mon Sep 17 00:00:00 2001 From: TheSola10 Date: Tue, 25 Feb 2025 11:59:21 +0100 Subject: [PATCH 22/22] Version bump to 2025.02-a2 --- default.nix | 2 +- nixie/__init__.py | 2 +- pyproject.toml | 2 +- static-bins/default.nix | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/default.nix b/default.nix index 96d165f..51b58e4 100644 --- a/default.nix +++ b/default.nix @@ -2,7 +2,7 @@ , libfaketime, ... }: let - nixie_ver = "2025.02-a1"; + nixie_ver = "2025.02-a2"; pzp = python3Packages.buildPythonPackage rec { pname = "pzp"; version = "0.0.27"; diff --git a/nixie/__init__.py b/nixie/__init__.py index 882629d..49b37e9 100644 --- a/nixie/__init__.py +++ b/nixie/__init__.py @@ -3,4 +3,4 @@ This is the Nixie tool for setting up Nixie in a Git repository. ''' -__version__ = '2025.02-a1' +__version__ = '2025.02-a2' diff --git a/pyproject.toml b/pyproject.toml index dbe5d27..899b49d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ artifacts = [ ] [tool.bumpver] -current_version = "2025.02-a1" +current_version = "2025.02-a2" version_pattern = "YYYY.0M-PYTAGNUM" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/static-bins/default.nix b/static-bins/default.nix index d311517..fc3b6ec 100644 --- a/static-bins/default.nix +++ b/static-bins/default.nix @@ -1,7 +1,7 @@ { nixpkgs ? # Nixpkgs import (from flake) -, nix-source ? builtins.fetchGit "https://github.com/nixos/nix" +, nix-source ? builtins.fetchGit "https://github.com/nixos/nix" # Nix packages source , fakedir ? builtins.fetchGit "https://github.com/thesola10/fakedir"