From 1077820edb0fcdb1e7e72f5a57134c13420bf4b9 Mon Sep 17 00:00:00 2001 From: Carl-Christian Sautter Date: Fri, 27 Mar 2026 20:41:09 +0000 Subject: [PATCH 01/10] refactor: move VM state into managed app data Store cache, .vagrant, and packer_cache under OS-specific app data paths and update build flows, helper scripts, packer templates, and docs. --- README.md | 18 ++- .../linux/mint/linux-mint-hyperv.pkr.hcl | 14 +- build/packer/linux/ubuntu/README.md | 15 ++- .../linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl | 14 +- .../ubuntu/linux-ubuntu-on-macos.pkr.hcl | 14 +- .../linux/ubuntu/linux-ubuntu-on-macos.sh | 15 ++- build/packer/windows/README.md | 28 ++-- .../packer/windows/windows11-on-macos.pkr.hcl | 28 ++-- build/packer/windows/windows11-on-macos.sh | 30 +++-- .../windows11-on-windows-hyperv.pkr.hcl | 18 ++- .../windows11-on-windows-virtualbox.pkr.hcl | 16 ++- deployments/utm/create-utm-vm.sh | 3 +- deployments/vagrant/ansible-windows/README.md | 24 +++- .../vagrant/linux-ubuntu-hyperv/README.md | 22 +++- .../0001-destroy-implementation-contract.md | 2 +- pkg/build/dependencies.go | 29 +++-- pkg/build/dependencies_test.go | 4 +- pkg/build/directories.go | 123 +++++++++++++++++- pkg/build/directories_test.go | 117 +++++++++++++++++ pkg/build/external_process_handler.go | 4 + pkg/build/generic_build.go | 1 + pkg/build/macos-silicon-build-helper_test.go | 15 ++- pkg/build/windows-build.go | 4 +- pkg/build/windows-linux-build.go | 4 +- pkg/deploy/windows-hyperv-deploy.go | 3 +- .../windows-hyperv-deploy_settings_test.go | 26 +++- pkg/deploy/windows-hyperv-destroy_test.go | 36 ++++- pkg/deploy/windows-hyperv-settings_test.go | 10 +- scripts/macos/README.md | 21 ++- scripts/macos/create-qemu-qcow2-disk.sh | 5 +- .../macos/create-win11-autounattend-iso.sh | 9 +- scripts/macos/download-arm64-uefi.sh | 19 ++- scripts/macos/download-utm-guest-tools.sh | 6 +- scripts/macos/download-virtio-win-iso.sh | 11 +- scripts/macos/playwright_win11_iso.py | 36 ++++- scripts/windows/download_win_11.ps1 | 17 ++- 36 files changed, 630 insertions(+), 131 deletions(-) create mode 100644 pkg/build/directories_test.go diff --git a/README.md b/README.md index 0b8a3bc1..2c409cc0 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,22 @@ To force a VM rebuild even when the cached build artifact already exists, use: go run cmd/main.go build windows11 --arch amd64 --no-cache ``` +#### Managed application data + +VM build and deployment state is now stored outside the repository in an OS-appropriate app-data directory: + +- macOS: `~/Library/Application Support/dev-alchemy` +- Windows: `%LOCALAPPDATA%\dev-alchemy` +- Linux: `${XDG_DATA_HOME:-~/.local/share}/dev-alchemy` + +Under that root, Dev Alchemy manages: + +- `cache/` for downloaded files and build artifacts +- `.vagrant/` for isolated Vagrant state +- `packer_cache/` for Packer plugin/download cache + +You can override the default location by setting `DEV_ALCHEMY_APP_DATA_DIR`. Dev Alchemy also exports `DEV_ALCHEMY_CACHE_DIR`, `DEV_ALCHEMY_VAGRANT_DIR`, and `DEV_ALCHEMY_PACKER_CACHE_DIR` for helper scripts and manual workflows. + ##### Enable ansible remote access on Windows @@ -343,7 +359,7 @@ go run cmd/main.go install You will need a Windows .iso file to use as the installation media for your virtual machine. You can download a Windows 10 or Windows Server .iso file from the Microsoft website. -Or use script to download a Windows 11 .iso file: [download_win_11.ps1](./scripts/windows/download_win_11.ps1) +Or use script to download a Windows 11 `.iso` file: [download_win_11.ps1](./scripts/windows/download_win_11.ps1). By default it stores the ISO under the managed cache directory described above. ##### Build a Windows VM diff --git a/build/packer/linux/mint/linux-mint-hyperv.pkr.hcl b/build/packer/linux/mint/linux-mint-hyperv.pkr.hcl index 15a8a175..27d8db20 100644 --- a/build/packer/linux/mint/linux-mint-hyperv.pkr.hcl +++ b/build/packer/linux/mint/linux-mint-hyperv.pkr.hcl @@ -22,9 +22,19 @@ variable "iso_checksum" { default = "sha256:759c9b5a2ad26eb9844b24f7da1696c705ff5fe07924a749f385f435176c2306" } +variable "cache_dir" { + type = string + default = env("DEV_ALCHEMY_CACHE_DIR") + description = "Managed cache directory outside the repository." + validation { + condition = var.cache_dir != "" + error_message = "cache_dir must be set, typically via DEV_ALCHEMY_CACHE_DIR." + } +} + source "hyperv-iso" "linuxmint" { vm_name = "linux-mint-packer-${formatdate("YYYY-MM-DD-hh-mm", timestamp())}" - output_directory = "${path.root}/../../../../cache/linux/hyperv-mint-output-${formatdate("YYYY-MM-DD-hh-mm", timestamp())}" + output_directory = "${var.cache_dir}/linux/hyperv-mint-output-${formatdate("YYYY-MM-DD-hh-mm", timestamp())}" iso_url = var.iso_url iso_checksum = var.iso_checksum @@ -64,7 +74,7 @@ build { } post-processor "vagrant" { - output = "${path.root}/../../../../cache/windows/linux-mint-hyperv.box" + output = "${var.cache_dir}/windows/linux-mint-hyperv.box" keep_input_artifact = false provider_override = "hyperv" compression_level = 1 diff --git a/build/packer/linux/ubuntu/README.md b/build/packer/linux/ubuntu/README.md index b956b6fe..0d773910 100644 --- a/build/packer/linux/ubuntu/README.md +++ b/build/packer/linux/ubuntu/README.md @@ -25,19 +25,25 @@ Both Hyper-V variants use cloud-init apt offline mode (`fallback: offline-instal Manual Packer usage: ```powershell +$AppDataDir = if ($env:DEV_ALCHEMY_APP_DATA_DIR) { $env:DEV_ALCHEMY_APP_DATA_DIR } else { Join-Path $env:LOCALAPPDATA "dev-alchemy" } +$CacheDir = Join-Path $AppDataDir "cache" +$env:DEV_ALCHEMY_CACHE_DIR = $CacheDir +$env:DEV_ALCHEMY_PACKER_CACHE_DIR = Join-Path $AppDataDir "packer_cache" +$isoPath = Join-Path $CacheDir "linux\ubuntu-24.04.3-live-server-amd64.iso" + packer init build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl # server -packer build -var "ubuntu_type=server" -var "iso_url=./cache/linux/ubuntu-24.04.3-live-server-amd64.iso" build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl +packer build -var "cache_dir=$CacheDir" -var "ubuntu_type=server" -var "iso_url=$isoPath" build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl # desktop -packer build -var "ubuntu_type=desktop" -var "iso_url=./cache/linux/ubuntu-24.04.3-live-server-amd64.iso" build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl +packer build -var "cache_dir=$CacheDir" -var "ubuntu_type=desktop" -var "iso_url=$isoPath" build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl ``` Output boxes: -- `cache/ubuntu/hyperv-ubuntu-server-amd64.box` -- `cache/ubuntu/hyperv-ubuntu-desktop-amd64.box` +- `%LOCALAPPDATA%\dev-alchemy\cache\ubuntu\hyperv-ubuntu-server-amd64.box` +- `%LOCALAPPDATA%\dev-alchemy\cache\ubuntu\hyperv-ubuntu-desktop-amd64.box` ## Build Ubuntu on macOS Hosts (UTM/QEMU) @@ -54,6 +60,7 @@ go run cmd/main.go build ubuntu --type desktop --arch $arch Manual script usage: ```bash +export DEV_ALCHEMY_APP_DATA_DIR="${DEV_ALCHEMY_APP_DATA_DIR:-$HOME/Library/Application Support/dev-alchemy}" build/packer/linux/ubuntu/linux-ubuntu-on-macos.sh --project-root "$PWD" --arch amd64 --ubuntu-type server build/packer/linux/ubuntu/linux-ubuntu-on-macos.sh --project-root "$PWD" --arch amd64 --ubuntu-type desktop ``` diff --git a/build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl b/build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl index 4ba621a9..e9124ed2 100644 --- a/build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl +++ b/build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl @@ -44,6 +44,16 @@ variable "switch_name" { default = "Default Switch" } +variable "cache_dir" { + type = string + default = env("DEV_ALCHEMY_CACHE_DIR") + description = "Managed cache directory outside the repository." + validation { + condition = var.cache_dir != "" + error_message = "cache_dir must be set, typically via DEV_ALCHEMY_CACHE_DIR." + } +} + locals { default_ubuntu_iso_url = "https://releases.ubuntu.com/${var.ubuntu_version}/ubuntu-${var.ubuntu_version}-live-server-amd64.iso" effective_iso_url = var.iso_url != "" ? var.iso_url : local.default_ubuntu_iso_url @@ -55,8 +65,8 @@ locals { "", "", ] - output_directory = "${path.root}/../../../../cache/linux/hyperv-ubuntu-${var.ubuntu_type}-output-${formatdate("YYYY-MM-DD-hh-mm", timestamp())}" - box_output = "${path.root}/../../../../cache/ubuntu/hyperv-ubuntu-${var.ubuntu_type}-amd64.box" + output_directory = "${var.cache_dir}/linux/hyperv-ubuntu-${var.ubuntu_type}-output-${formatdate("YYYY-MM-DD-hh-mm", timestamp())}" + box_output = "${var.cache_dir}/ubuntu/hyperv-ubuntu-${var.ubuntu_type}-amd64.box" } source "hyperv-iso" "ubuntu" { diff --git a/build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl b/build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl index 95a48e1b..bc9d9e42 100644 --- a/build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl +++ b/build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl @@ -64,10 +64,20 @@ variable "memory" { description = "Memory in MB to allocate to the VM" } +variable "cache_dir" { + type = string + default = env("DEV_ALCHEMY_CACHE_DIR") + description = "Managed cache directory outside the repository." + validation { + condition = var.cache_dir != "" + error_message = "cache_dir must be set, typically via DEV_ALCHEMY_CACHE_DIR." + } +} + locals { iso_url = var.iso_url ubuntu_iso_checksum = var.arch == "amd64" ? "sha256:c3514bf0056180d09376462a7a1b4f213c1d6e8ea67fae5c25099c6fd3d8274b" : "none" - cache_directory = "${path.root}/../../../../cache" + cache_directory = var.cache_dir boot_command = { "amd64" = [ "e", @@ -100,7 +110,7 @@ locals { ["-accel", var.is_ci ? "tcg,thread=multi,tb-size=512" : "hvf"], ["-machine", "virt,highmem=on"], ["-cpu", var.is_ci ? "max,sve=off,pauth-impdef=on" : "host"], - ["-bios", "${path.root}/../../../../cache/qemu-uefi/usr/share/qemu-efi-aarch64/QEMU_EFI.fd"], + ["-bios", "${local.cache_directory}/qemu-uefi/usr/share/qemu-efi-aarch64/QEMU_EFI.fd"], ["-device", "ramfb"], ["-smp", "cpus=${var.cpus},cores=${var.cpus},sockets=1,threads=1"], ["-global", "PIIX4_PM.disable_s3=1"], diff --git a/build/packer/linux/ubuntu/linux-ubuntu-on-macos.sh b/build/packer/linux/ubuntu/linux-ubuntu-on-macos.sh index b03cea1d..e388bc11 100644 --- a/build/packer/linux/ubuntu/linux-ubuntu-on-macos.sh +++ b/build/packer/linux/ubuntu/linux-ubuntu-on-macos.sh @@ -22,6 +22,9 @@ project_root=$( cd "${script_dir}/../../../.." pwd -P ) +app_data_dir="${DEV_ALCHEMY_APP_DATA_DIR:-$HOME/Library/Application Support/dev-alchemy}" +cache_dir="${DEV_ALCHEMY_CACHE_DIR:-$app_data_dir/cache}" +packer_cache_dir="${DEV_ALCHEMY_PACKER_CACHE_DIR:-$app_data_dir/packer_cache}" while [[ $# -gt 0 ]]; do case "$1" in @@ -95,7 +98,11 @@ while [[ $# -gt 0 ]]; do esac done -cache_dir="$project_root/cache" +mkdir -p "$cache_dir" "$packer_cache_dir" +export DEV_ALCHEMY_APP_DATA_DIR="$app_data_dir" +export DEV_ALCHEMY_CACHE_DIR="$cache_dir" +export DEV_ALCHEMY_PACKER_CACHE_DIR="$packer_cache_dir" +export PACKER_CACHE_DIR="$packer_cache_dir" # Download uefi-firmware if it doesn't exist if [ "$arch" = "arm64" ]; then @@ -103,9 +110,9 @@ if [ "$arch" = "arm64" ]; then fi # Download the Ubuntu ISO if it doesn't exist -iso_path="$project_root/cache/linux/ubuntu-24.04.3-live-server-amd64.iso" +iso_path="$cache_dir/linux/ubuntu-24.04.3-live-server-amd64.iso" if [ "$arch" = "arm64" ]; then - iso_path="$project_root/cache/linux/ubuntu-24.04.3-live-server-arm64.iso" + iso_path="$cache_dir/linux/ubuntu-24.04.3-live-server-arm64.iso" iso_url="https://cdimage.ubuntu.com/releases/24.04.3/release/ubuntu-24.04.3-live-server-arm64.iso" iso_checksum="2ee2163c9b901ff5926400e80759088ff3b879982a3956c02100495b489fd555" mkdir -p "$(dirname "$iso_path")" @@ -159,4 +166,4 @@ packer init "build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl" if [ "$verbose" = "true" ]; then export PACKER_LOG=1 fi -packer build -var "iso_url=$iso_path" -var "ubuntu_type=$ubuntu_type" -var "headless=$headless" -var "vnc_port=$vnc_port" -var "arch=$arch" -var "cpus=$cpus" -var "memory=$memory" "build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl" +packer build -var "cache_dir=$cache_dir" -var "iso_url=$iso_path" -var "ubuntu_type=$ubuntu_type" -var "headless=$headless" -var "vnc_port=$vnc_port" -var "arch=$arch" -var "cpus=$cpus" -var "memory=$memory" "build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl" diff --git a/build/packer/windows/README.md b/build/packer/windows/README.md index 5df86762..430eeb56 100644 --- a/build/packer/windows/README.md +++ b/build/packer/windows/README.md @@ -2,7 +2,7 @@ ## Build Windows on Windows Hosts -This directory contains a Packer template for building Windows images. +This directory contains the Packer templates for building Windows 11 images on Windows and macOS hosts. ### Prerequisites @@ -12,27 +12,32 @@ This directory contains a Packer template for building Windows images. ### Usage -Set the iso_url variable in [windows.pkr.hcl](windows.pkr.hcl) to point to your Windows ISO file. +For manual builds on Windows, use the current Hyper-V or VirtualBox templates and point them at the managed Dev Alchemy cache. ```powershell -# Example for Windows 11 ISO -$isoPath = "C:\path\to\your\Win11_*.iso" +$AppDataDir = if ($env:DEV_ALCHEMY_APP_DATA_DIR) { $env:DEV_ALCHEMY_APP_DATA_DIR } else { Join-Path $env:LOCALAPPDATA "dev-alchemy" } +$CacheDir = Join-Path $AppDataDir "cache" +$env:DEV_ALCHEMY_CACHE_DIR = $CacheDir +$env:DEV_ALCHEMY_PACKER_CACHE_DIR = Join-Path $AppDataDir "packer_cache" -# Find newest iso file in cache/windows directory -$isoPath = Get-ChildItem -Path ".\cache\windows" -Filter "Win11_*.iso" | Sort-Object LastWriteTime -Descending | Select-Object -First 1 | Select-Object -ExpandProperty FullName +# Find newest ISO file in the managed cache +$isoPath = Get-ChildItem -Path (Join-Path $CacheDir "windows11\iso") -Filter "Win11_*.iso" | Sort-Object LastWriteTime -Descending | Select-Object -First 1 | Select-Object -ExpandProperty FullName Write-Host "Using ISO: $isoPath" ``` To build the Windows image, run: ```powershell -# with default iso_url from windows.pkr.hcl -packer build build/packer/windows/windows.pkr.hcl -# or override iso_url -packer build -var "iso_url=$isoPath" build/packer/windows/windows.pkr.hcl +# Hyper-V +packer init build/packer/windows/windows11-on-windows-hyperv.pkr.hcl +packer build -var "cache_dir=$CacheDir" -var "iso_url=$isoPath" build/packer/windows/windows11-on-windows-hyperv.pkr.hcl + +# VirtualBox +packer init build/packer/windows/windows11-on-windows-virtualbox.pkr.hcl +packer build -var "cache_dir=$CacheDir" -var "iso_url=$isoPath" build/packer/windows/windows11-on-windows-virtualbox.pkr.hcl ``` -You can reduce build time by disabling compression in the Vagrant post-processor. Edit the `compression_level` in the `post-processor "vagrant"` block of [windows.pkr.hcl](windows.pkr.hcl) and set it to `0` for no compression. +You can reduce build time by disabling compression in the `post-processor "vagrant"` block of the relevant template and setting `compression_level = 0`. Default for packer is `6`. [Compression Level Reference](https://developer.hashicorp.com/packer/docs/post-processors/compress#compression_level) @@ -57,6 +62,7 @@ The process is idempotent, so you can re-run commands without issues. ```bash arch=arm64 # or amd64 +export DEV_ALCHEMY_APP_DATA_DIR="${DEV_ALCHEMY_APP_DATA_DIR:-$HOME/Library/Application Support/dev-alchemy}" go run cmd/main.go install go run cmd/main.go build windows11 --arch $arch go run cmd/main.go create windows11 --arch $arch diff --git a/build/packer/windows/windows11-on-macos.pkr.hcl b/build/packer/windows/windows11-on-macos.pkr.hcl index 54ea69ec..bfd73229 100644 --- a/build/packer/windows/windows11-on-macos.pkr.hcl +++ b/build/packer/windows/windows11-on-macos.pkr.hcl @@ -45,22 +45,32 @@ variable "memory" { description = "Memory in MB to allocate to the VM" } +variable "cache_dir" { + type = string + default = env("DEV_ALCHEMY_CACHE_DIR") + description = "Managed cache directory outside the repository." + validation { + condition = var.cache_dir != "" + error_message = "cache_dir must be set, typically via DEV_ALCHEMY_CACHE_DIR." + } +} + variable "is_ci" { type = bool default = env("CI") == "true" } locals { - cache_directory = "${path.root}/../../../cache" + cache_directory = var.cache_dir win11_default_iso = { - amd64 = "../../../cache/windows11/iso/win11_25h2_english_amd64.iso" - arm64 = "../../../cache/windows11/iso/Win11_25H2_English_arm64.iso" + amd64 = "${local.cache_directory}/windows11/iso/win11_25h2_english_amd64.iso" + arm64 = "${local.cache_directory}/windows11/iso/Win11_25H2_English_arm64.iso" } win11_iso = var.iso_url != "" ? var.iso_url : local.win11_default_iso[var.arch] win11_qcow2 = "${local.cache_directory}/windows11/qemu-windows11-${var.arch}.qcow2" - win11_guest_tools = "${path.root}/../../../cache/utm/utm-guest-tools-latest.iso" - win11_virtio_iso = "${path.root}/../../../cache/windows/virtio-win.iso" - win11_uefi_bios = "${path.root}/../../../cache/qemu-uefi/usr/share/qemu-efi-aarch64/QEMU_EFI.fd" + win11_guest_tools = "${local.cache_directory}/utm/utm-guest-tools-latest.iso" + win11_virtio_iso = "${local.cache_directory}/windows/virtio-win.iso" + win11_uefi_bios = "${local.cache_directory}/qemu-uefi/usr/share/qemu-efi-aarch64/QEMU_EFI.fd" qemu_args = { "amd64" = [ ["-device", "qemu-xhci,id=usb"], @@ -81,7 +91,7 @@ locals { ["-cpu", var.is_ci ? "max,sve=off,pauth-impdef=on" : "host"], # setting a specific cpu model leads to issues, therefore using max above # ["-cpu", var.is_ci ? "cortex-a72" : "host"], - ["-bios", "${path.root}/../../../cache/qemu-uefi/usr/share/qemu-efi-aarch64/QEMU_EFI.fd"], + ["-bios", "${local.win11_uefi_bios}"], ["-device", "ramfb"], ["-device", "qemu-xhci"], ["-device", "usb-kbd"], @@ -89,9 +99,9 @@ locals { ["-device", "usb-storage,drive=install,removable=true,bootindex=0"], ["-drive", "if=none,id=install,format=raw,media=cdrom,file=${local.win11_iso},readonly=true"], ["-device", "usb-storage,drive=virtio-drivers,removable=true,bootindex=2"], - ["-drive", "if=none,id=virtio-drivers,format=raw,media=cdrom,file=${path.root}/../../../cache/windows/virtio-win.iso,readonly=true"], + ["-drive", "if=none,id=virtio-drivers,format=raw,media=cdrom,file=${local.win11_virtio_iso},readonly=true"], ["-device", "usb-storage,drive=utm-tools,removable=true,bootindex=3"], - ["-drive", "if=none,id=utm-tools,format=raw,media=cdrom,file=${path.root}/../../../cache/utm/utm-guest-tools-latest.iso,readonly=true"], + ["-drive", "if=none,id=utm-tools,format=raw,media=cdrom,file=${local.win11_guest_tools},readonly=true"], ["-device", "nvme,drive=nvme0,serial=deadbeef,bootindex=1"], ["-drive", "if=none,media=disk,id=nvme0,format=qcow2,file.filename=${local.cache_directory}/windows11/qemu-windows11-arm64.qcow2,discard=unmap,detect-zeroes=unmap"], ["-boot", "order=c,menu=on"], diff --git a/build/packer/windows/windows11-on-macos.sh b/build/packer/windows/windows11-on-macos.sh index 5a113419..b1d445d3 100644 --- a/build/packer/windows/windows11-on-macos.sh +++ b/build/packer/windows/windows11-on-macos.sh @@ -20,6 +20,9 @@ project_root=$( cd "${script_dir}/../../.." pwd -P ) +app_data_dir="${DEV_ALCHEMY_APP_DATA_DIR:-$HOME/Library/Application Support/dev-alchemy}" +cache_dir="${DEV_ALCHEMY_CACHE_DIR:-$app_data_dir/cache}" +packer_cache_dir="${DEV_ALCHEMY_PACKER_CACHE_DIR:-$app_data_dir/packer_cache}" while [[ $# -gt 0 ]]; do case "$1" in @@ -88,12 +91,17 @@ echo "Using architecture: $arch" echo "Headless mode: $headless" cd "${project_root}" || exit 1 +mkdir -p "$cache_dir" "$packer_cache_dir" +export DEV_ALCHEMY_APP_DATA_DIR="$app_data_dir" +export DEV_ALCHEMY_CACHE_DIR="$cache_dir" +export DEV_ALCHEMY_PACKER_CACHE_DIR="$packer_cache_dir" +export PACKER_CACHE_DIR="$packer_cache_dir" # download the Windows 11 ISO if not already present -if [ ! -d ./cache/windows11/iso ]; then - mkdir -p ./cache/windows11/iso +if [ ! -d "$cache_dir/windows11/iso" ]; then + mkdir -p "$cache_dir/windows11/iso" fi -if [ ! -f "./cache/windows11/iso/win11_25H2_english_$arch.iso" ]; then +if [ ! -f "$cache_dir/windows11/iso/win11_25h2_english_$arch.iso" ]; then echo "Downloading Windows 11 $arch ISO" cd "${project_root}/scripts/macos/" || exit 1 if [ ! -d .venv ]; then @@ -112,15 +120,15 @@ if [ ! -f "./cache/windows11/iso/win11_25H2_english_$arch.iso" ]; then elif [ "$arch" = "arm64" ]; then python playwright_win11_iso.py --arm fi - mkdir -p "${project_root}/cache/windows11/iso" - cd "${project_root}/cache/windows/" || exit 1 + mkdir -p "${cache_dir}/windows11/iso" + cd "${cache_dir}/windows/" || exit 1 if [ "$headless" = "true" ]; then echo "Running in headless mode, skipping ISO download progress bar" - curl -o "${project_root}/cache/windows11/iso/win11_25h2_english_$arch.iso" "$(cat "./win11_${arch}_iso_url.txt")" + curl -o "${cache_dir}/windows11/iso/win11_25h2_english_$arch.iso" "$(cat "./win11_${arch}_iso_url.txt")" else echo "Running in interactive mode, showing ISO download progress bar" - curl --progress-bar -o "${project_root}/cache/windows11/iso/win11_25h2_english_$arch.iso" "$(cat "./win11_${arch}_iso_url.txt")" + curl --progress-bar -o "${cache_dir}/windows11/iso/win11_25h2_english_$arch.iso" "$(cat "./win11_${arch}_iso_url.txt")" fi cd "${project_root}" || exit 1 @@ -151,14 +159,14 @@ packer init "${project_root}/build/packer/windows/windows11-on-macos.pkr.hcl" # determine the Windows 11 ISO path to use if [ "$arch" = "amd64" ]; then - win11_iso_path="${project_root}/cache/windows11/iso/win11_25h2_english_$arch.iso" + win11_iso_path="${cache_dir}/windows11/iso/win11_25h2_english_$arch.iso" elif [ "$arch" = "arm64" ]; then # use the unattended ISO we created earlier - win11_iso_path="${project_root}/cache/windows11/iso/Win11_ARM64_Unattended.iso" + win11_iso_path="${cache_dir}/windows11/iso/Win11_ARM64_Unattended.iso" fi # remove packer output directory if it exists -output_dir="${project_root}/cache/windows11/qemu-out-windows11-${arch}" +output_dir="${cache_dir}/windows11/qemu-out-windows11-${arch}" if [ -d "$output_dir" ]; then echo "Removing existing Packer output directory..." rm -rf "$output_dir" @@ -167,7 +175,7 @@ fi if [ "$verbose" = "true" ]; then export PACKER_LOG=1 fi -packer build -var "iso_url=${win11_iso_path}" -var "headless=$headless" -var "vnc_port=$vnc_port" -var "arch=$arch" -var "cpus=$cpus" -var "memory=$memory" "build/packer/windows/windows11-on-macos.pkr.hcl" +packer build -var "cache_dir=${cache_dir}" -var "iso_url=${win11_iso_path}" -var "headless=$headless" -var "vnc_port=$vnc_port" -var "arch=$arch" -var "cpus=$cpus" -var "memory=$memory" "build/packer/windows/windows11-on-macos.pkr.hcl" packer_exit_code=$? exit $packer_exit_code diff --git a/build/packer/windows/windows11-on-windows-hyperv.pkr.hcl b/build/packer/windows/windows11-on-windows-hyperv.pkr.hcl index 2b19508b..3d58b8c0 100644 --- a/build/packer/windows/windows11-on-windows-hyperv.pkr.hcl +++ b/build/packer/windows/windows11-on-windows-hyperv.pkr.hcl @@ -14,7 +14,7 @@ packer { variable "iso_url" { type = string - default = "../../../cache/windows11/iso/Win11_25H2_English_x64.iso" + default = "" } variable "cpus" { @@ -34,13 +34,23 @@ variable "temp_disk_path" { description = "Path to use for temporary files and VM storage (e.g., D:\\ for Azure local temp disk)" } +variable "cache_dir" { + type = string + default = env("DEV_ALCHEMY_CACHE_DIR") + description = "Managed cache directory outside the repository." + validation { + condition = var.cache_dir != "" + error_message = "cache_dir must be set, typically via DEV_ALCHEMY_CACHE_DIR." + } +} + locals { - temp_dir = var.temp_disk_path != "" ? var.temp_disk_path : "${path.root}/../../../cache/windows11/hyperv-temp" + temp_dir = var.temp_disk_path != "" ? var.temp_disk_path : "${var.cache_dir}/windows11/hyperv-temp" } source "hyperv-iso" "win11" { vm_name = "win11-packer-${formatdate("YYYY-MM-DD-hh-mm", timestamp())}" - output_directory = "${path.root}/../../../cache/windows11/hyperv-output-${formatdate("YYYY-MM-DD-hh-mm", timestamp())}" + output_directory = "${var.cache_dir}/windows11/hyperv-output-${formatdate("YYYY-MM-DD-hh-mm", timestamp())}" temp_path = local.temp_dir iso_url = var.iso_url @@ -86,7 +96,7 @@ build { sources = ["source.hyperv-iso.win11"] post-processor "vagrant" { - output = "${path.root}/../../../cache/windows11/hyperv-windows11-amd64.box" + output = "${var.cache_dir}/windows11/hyperv-windows11-amd64.box" keep_input_artifact = false provider_override = "hyperv" compression_level = 1 diff --git a/build/packer/windows/windows11-on-windows-virtualbox.pkr.hcl b/build/packer/windows/windows11-on-windows-virtualbox.pkr.hcl index a423c628..4e7fd7b5 100644 --- a/build/packer/windows/windows11-on-windows-virtualbox.pkr.hcl +++ b/build/packer/windows/windows11-on-windows-virtualbox.pkr.hcl @@ -14,7 +14,7 @@ packer { variable "iso_url" { type = string - default = "../../../cache/windows11/iso/Win11_25H2_English_x64.iso" + default = "" } variable "nested_virt" { @@ -40,9 +40,19 @@ variable "temp_disk_path" { description = "Path to use for temporary files and VM storage (e.g., D:\\ for Azure local temp disk)" } +variable "cache_dir" { + type = string + default = env("DEV_ALCHEMY_CACHE_DIR") + description = "Managed cache directory outside the repository." + validation { + condition = var.cache_dir != "" + error_message = "cache_dir must be set, typically via DEV_ALCHEMY_CACHE_DIR." + } +} + source "virtualbox-iso" "win11" { vm_name = "win11-packer-${formatdate("YYYY-MM-DD-hh-mm", timestamp())}" - output_directory = var.temp_disk_path != "" ? var.temp_disk_path : "${path.root}/../../../cache/windows11/virtualbox-output-${formatdate("YYYY-MM-DD-hh-mm", timestamp())}" + output_directory = var.temp_disk_path != "" ? var.temp_disk_path : "${var.cache_dir}/windows11/virtualbox-output-${formatdate("YYYY-MM-DD-hh-mm", timestamp())}" iso_url = var.iso_url iso_checksum = "none" @@ -98,7 +108,7 @@ build { sources = ["source.virtualbox-iso.win11"] post-processor "vagrant" { - output = "${path.root}/../../../cache/windows11/virtualbox-windows11-amd64.box" + output = "${var.cache_dir}/windows11/virtualbox-windows11-amd64.box" keep_input_artifact = false provider_override = "virtualbox" compression_level = 1 diff --git a/deployments/utm/create-utm-vm.sh b/deployments/utm/create-utm-vm.sh index bcb30919..deb93eb1 100644 --- a/deployments/utm/create-utm-vm.sh +++ b/deployments/utm/create-utm-vm.sh @@ -50,7 +50,8 @@ project_root=$( pwd ) -cache_dir="$project_root/cache" +app_data_dir="${DEV_ALCHEMY_APP_DATA_DIR:-$HOME/Library/Application Support/dev-alchemy}" +cache_dir="${DEV_ALCHEMY_CACHE_DIR:-$app_data_dir/cache}" utm_vm_dir="/Users/$(whoami)/Library/Containers/com.utmapp.UTM/Data/Documents" diff --git a/deployments/vagrant/ansible-windows/README.md b/deployments/vagrant/ansible-windows/README.md index f1a851cd..5ac7da08 100644 --- a/deployments/vagrant/ansible-windows/README.md +++ b/deployments/vagrant/ansible-windows/README.md @@ -3,6 +3,14 @@ This guide will help you set up and run Ansible playbooks on a Windows VM using Vagrant with Hyper-V as the provider. All commands are meant to be run in a powershell terminal on a Windows host machine. +Managed Dev Alchemy paths on Windows default to: + +- App data root: `%LOCALAPPDATA%\dev-alchemy` +- Build cache: `%LOCALAPPDATA%\dev-alchemy\cache` +- Vagrant state: `%LOCALAPPDATA%\dev-alchemy\.vagrant` + +Set `DEV_ALCHEMY_APP_DATA_DIR` if you want to override the default root. + ## Prerequisites Run the dependency installer from repository root in an elevated PowerShell session: @@ -22,7 +30,7 @@ Ensure you have the following installed: ## Adding the Vagrant Box Load the Vagrant box and start the VM using Hyper-V as the provider. -The box artifact is expected at `.\cache\windows11\hyperv-windows11-amd64.box`. +The box artifact is expected at `%LOCALAPPDATA%\dev-alchemy\cache\windows11\hyperv-windows11-amd64.box`. Set `VAGRANT_HYPERV_SWITCH` to avoid Hyper-V bridge selection prompts during `vagrant up`: @@ -32,8 +40,12 @@ $env:VAGRANT_HYPERV_SWITCH = "Default Switch" Then add the box and boot the VM: -```bash -vagrant box add win11-packer .\cache\windows11\hyperv-windows11-amd64.box --provider hyperv +```powershell +$AppDataDir = if ($env:DEV_ALCHEMY_APP_DATA_DIR) { $env:DEV_ALCHEMY_APP_DATA_DIR } else { Join-Path $env:LOCALAPPDATA "dev-alchemy" } +$CacheDir = Join-Path $AppDataDir "cache" +$VagrantRoot = Join-Path $AppDataDir ".vagrant" +$env:VAGRANT_DOTFILE_PATH = Join-Path $VagrantRoot "win11-packer" +vagrant box add win11-packer (Join-Path $CacheDir "windows11\hyperv-windows11-amd64.box") --provider hyperv vagrant up --provider hyperv ``` @@ -46,7 +58,7 @@ After the VM is up, you can connect to it using Hyper-V Manager or via RDP. The The provisioning wrapper discovers the VM IP automatically, but you can inspect it manually: -```bash +```powershell vagrant winrm -c "ipconfig" ``` @@ -93,7 +105,7 @@ If `CYGWIN_TERMINAL_PATH` points to `mintty.exe`, provisioning resolves it to th After installing host dependencies, run provisioning from the repository root. The wrapper resolves IP address via `vagrant winrm -c ipconfig` and runs `ansible-playbook` through Cygwin. -```bash +```powershell go run cmd/main.go provision windows11 --arch amd64 --check go run cmd/main.go provision windows11 --arch amd64 ``` @@ -102,7 +114,7 @@ go run cmd/main.go provision windows11 --arch amd64 When you are done, you can destroy the Vagrant box and remove the box from your system: -```bash +```powershell vagrant destroy vagrant box remove win11-packer --provider hyperv ``` diff --git a/deployments/vagrant/linux-ubuntu-hyperv/README.md b/deployments/vagrant/linux-ubuntu-hyperv/README.md index 90013a5f..9d1ddabe 100644 --- a/deployments/vagrant/linux-ubuntu-hyperv/README.md +++ b/deployments/vagrant/linux-ubuntu-hyperv/README.md @@ -8,6 +8,14 @@ This guide covers the Windows-host workflow for Ubuntu Hyper-V with the Go wrapp All commands are intended for PowerShell on a Windows host. +Managed Dev Alchemy paths on Windows default to: + +- App data root: `%LOCALAPPDATA%\dev-alchemy` +- Build cache: `%LOCALAPPDATA%\dev-alchemy\cache` +- Vagrant state: `%LOCALAPPDATA%\dev-alchemy\.vagrant` + +Set `DEV_ALCHEMY_APP_DATA_DIR` if you want to override the default root. + ## Prerequisites - Run the dependency installer from repository root in an elevated PowerShell session: @@ -35,8 +43,8 @@ go run cmd/main.go build ubuntu --type desktop --arch amd64 Expected artifacts: -- `cache/ubuntu/hyperv-ubuntu-server-amd64.box` -- `cache/ubuntu/hyperv-ubuntu-desktop-amd64.box` +- `%LOCALAPPDATA%\dev-alchemy\cache\ubuntu\hyperv-ubuntu-server-amd64.box` +- `%LOCALAPPDATA%\dev-alchemy\cache\ubuntu\hyperv-ubuntu-desktop-amd64.box` ## Create/Start the VM @@ -102,10 +110,14 @@ $env:CYGWIN_TERMINAL_PATH = "C:\tools\cygwin\bin\mintty.exe" If you want to run Vagrant directly: ```powershell +$AppDataDir = if ($env:DEV_ALCHEMY_APP_DATA_DIR) { $env:DEV_ALCHEMY_APP_DATA_DIR } else { Join-Path $env:LOCALAPPDATA "dev-alchemy" } +$CacheDir = Join-Path $AppDataDir "cache" +$VagrantRoot = Join-Path $AppDataDir ".vagrant" $type = "server" # or "desktop" $env:VAGRANT_BOX_NAME = "linux-ubuntu-$type-packer" $env:VAGRANT_VM_NAME = "linux-ubuntu-$type-packer" -vagrant box add $env:VAGRANT_BOX_NAME ".\cache\ubuntu\hyperv-ubuntu-$type-amd64.box" --provider hyperv --force +$env:VAGRANT_DOTFILE_PATH = Join-Path $VagrantRoot $env:VAGRANT_VM_NAME +vagrant box add $env:VAGRANT_BOX_NAME (Join-Path $CacheDir "ubuntu\hyperv-ubuntu-$type-amd64.box") --provider hyperv --force cd deployments\vagrant\linux-ubuntu-hyperv vagrant up --provider hyperv cd ..\..\.. @@ -114,9 +126,13 @@ cd ..\..\.. ## Destroy and Cleanup ```powershell +$AppDataDir = if ($env:DEV_ALCHEMY_APP_DATA_DIR) { $env:DEV_ALCHEMY_APP_DATA_DIR } else { Join-Path $env:LOCALAPPDATA "dev-alchemy" } +$VagrantRoot = Join-Path $AppDataDir ".vagrant" +$env:VAGRANT_DOTFILE_PATH = Join-Path $VagrantRoot "linux-ubuntu-server-packer" cd deployments\vagrant\linux-ubuntu-hyperv vagrant destroy -f vagrant box remove linux-ubuntu-server-packer --provider hyperv +$env:VAGRANT_DOTFILE_PATH = Join-Path $VagrantRoot "linux-ubuntu-desktop-packer" vagrant box remove linux-ubuntu-desktop-packer --provider hyperv cd ..\..\.. ``` diff --git a/docs/adr/0001-destroy-implementation-contract.md b/docs/adr/0001-destroy-implementation-contract.md index 95000460..2162954d 100644 --- a/docs/adr/0001-destroy-implementation-contract.md +++ b/docs/adr/0001-destroy-implementation-contract.md @@ -22,7 +22,7 @@ Destroy implementations must be: - Deterministic: derive the resource identity directly from `VirtualMachineConfig` and documented environment overrides. - Idempotent: return success when the VM is already absent. -- Host-local: remove only the resources created by `alchemy create`; do not delete build artifacts from `cache/`. +- Host-local: remove only the resources created by `alchemy create`; do not delete build artifacts from the managed app-data cache. - Routed through the shared dispatcher in [pkg/deploy/destroy.go](/workspaces/dev-alchemy/pkg/deploy/destroy.go). ## Required Changes When Adding A VM Config diff --git a/pkg/build/dependencies.go b/pkg/build/dependencies.go index 1334e448..103254b3 100644 --- a/pkg/build/dependencies.go +++ b/pkg/build/dependencies.go @@ -255,6 +255,7 @@ func getWindows11Download(arch string, savePath string, download bool) (string, WorkingDir: workdir, ExecutablePath: venvPython, Args: args, + Env: GetDirectoriesInstance().ManagedEnv(), Timeout: 10 * time.Minute, } if _, err := RunExternalProcess(config); err != nil { @@ -265,8 +266,8 @@ func getWindows11Download(arch string, savePath string, download bool) (string, return "", nil } - // #nosec G304 -- url_file is selected from a fixed arch allowlist and resolved under the repo cache directory. - content, err := os.ReadFile(filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/windows/"+url_file)) + // #nosec G304 -- url_file is selected from a fixed arch allowlist and resolved under the managed cache directory. + content, err := os.ReadFile(GetDirectoriesInstance().CachePath("windows", url_file)) if err != nil { return "", err } @@ -295,7 +296,7 @@ func isSafeDebianPackageSegment(value string) bool { func getWebFileDependencies() []WebFileDependency { return []WebFileDependency{ { - LocalPath: filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/utm/utm-guest-tools-latest.iso"), + LocalPath: GetDirectoriesInstance().CachePath("utm", "utm-guest-tools-latest.iso"), Checksum: "", Source: "https://getutm.app/downloads/utm-guest-tools-latest.iso", RelatedVmConfigs: []VirtualMachineConfig{ @@ -314,10 +315,10 @@ func getWebFileDependencies() []WebFileDependency { }, }, { - LocalPath: filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/windows11/iso/win11_25h2_english_amd64.iso"), + LocalPath: GetDirectoriesInstance().CachePath("windows11", "iso", "win11_25h2_english_amd64.iso"), Checksum: "", BeforeHook: func() (string, error) { - return getWindows11Download("amd64", filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/windows11/iso/win11_25h2_english_amd64.iso"), false) + return getWindows11Download("amd64", GetDirectoriesInstance().CachePath("windows11", "iso", "win11_25h2_english_amd64.iso"), false) }, RelatedVmConfigs: []VirtualMachineConfig{ { @@ -347,10 +348,10 @@ func getWebFileDependencies() []WebFileDependency { }, }, { - LocalPath: filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/windows11/iso/win11_25h2_english_arm64.iso"), + LocalPath: GetDirectoriesInstance().CachePath("windows11", "iso", "win11_25h2_english_arm64.iso"), Checksum: "", BeforeHook: func() (string, error) { - return getWindows11Download("arm64", filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/windows11/iso/win11_25h2_english_arm64.iso"), false) + return getWindows11Download("arm64", GetDirectoriesInstance().CachePath("windows11", "iso", "win11_25h2_english_arm64.iso"), false) }, RelatedVmConfigs: []VirtualMachineConfig{ { @@ -362,7 +363,7 @@ func getWebFileDependencies() []WebFileDependency { }, }, { - LocalPath: filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/qemu-efi-aarch64_all.deb"), + LocalPath: GetDirectoriesInstance().CachePath("qemu-efi-aarch64_all.deb"), Checksum: "", BeforeHook: func() (string, error) { return resolveDebianPackageURL("trixie", "qemu-efi-aarch64") @@ -377,7 +378,7 @@ func getWebFileDependencies() []WebFileDependency { }, }, { - LocalPath: filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/windows/virtio-win.iso"), + LocalPath: GetDirectoriesInstance().CachePath("windows", "virtio-win.iso"), Checksum: "", Source: "https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/archive-virtio/virtio-win-0.1.266-1/virtio-win-0.1.266.iso", RelatedVmConfigs: []VirtualMachineConfig{ @@ -390,7 +391,7 @@ func getWebFileDependencies() []WebFileDependency { }, }, { - LocalPath: filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/linux/ubuntu-24.04.3-live-server-arm64.iso"), + LocalPath: GetDirectoriesInstance().CachePath("linux", "ubuntu-24.04.3-live-server-arm64.iso"), Checksum: "sha256:2ee2163c9b901ff5926400e80759088ff3b879982a3956c02100495b489fd555", Source: "https://cdimage.ubuntu.com/releases/24.04.3/release/ubuntu-24.04.3-live-server-arm64.iso", RelatedVmConfigs: []VirtualMachineConfig{ @@ -404,7 +405,7 @@ func getWebFileDependencies() []WebFileDependency { }, }, { - LocalPath: filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/linux/ubuntu-24.04.3-live-server-amd64.iso"), + LocalPath: GetDirectoriesInstance().CachePath("linux", "ubuntu-24.04.3-live-server-amd64.iso"), Checksum: "sha256:c3514bf0056180d09376462a7a1b4f213c1d6e8ea67fae5c25099c6fd3d8274b", Source: "https://releases.ubuntu.com/24.04.3/ubuntu-24.04.3-live-server-amd64.iso", RelatedVmConfigs: []VirtualMachineConfig{ @@ -425,7 +426,7 @@ func getWebFileDependencies() []WebFileDependency { }, }, { - LocalPath: filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/linux/ubuntu-24.04.3-live-server-arm64.iso"), + LocalPath: GetDirectoriesInstance().CachePath("linux", "ubuntu-24.04.3-live-server-arm64.iso"), Checksum: "sha256:2ee2163c9b901ff5926400e80759088ff3b879982a3956c02100495b489fd555", Source: "https://cdimage.ubuntu.com/releases/24.04.3/release/ubuntu-24.04.3-live-server-arm64.iso", RelatedVmConfigs: []VirtualMachineConfig{ @@ -439,7 +440,7 @@ func getWebFileDependencies() []WebFileDependency { }, }, { - LocalPath: filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/linux/ubuntu-24.04.3-live-server-amd64.iso"), + LocalPath: GetDirectoriesInstance().CachePath("linux", "ubuntu-24.04.3-live-server-amd64.iso"), Checksum: "sha256:c3514bf0056180d09376462a7a1b4f213c1d6e8ea67fae5c25099c6fd3d8274b", Source: "https://releases.ubuntu.com/24.04.3/ubuntu-24.04.3-live-server-amd64.iso", RelatedVmConfigs: []VirtualMachineConfig{ @@ -460,7 +461,7 @@ func getWebFileDependencies() []WebFileDependency { }, }, { - LocalPath: filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/qemu-efi-aarch64_all.deb"), + LocalPath: GetDirectoriesInstance().CachePath("qemu-efi-aarch64_all.deb"), Checksum: "", BeforeHook: func() (string, error) { return resolveDebianPackageURL("trixie", "qemu-efi-aarch64") diff --git a/pkg/build/dependencies_test.go b/pkg/build/dependencies_test.go index 5e999f55..6d6e3f3f 100644 --- a/pkg/build/dependencies_test.go +++ b/pkg/build/dependencies_test.go @@ -92,7 +92,7 @@ func TestGetWindows11DownloadAmd64(t *testing.T) { if os.Getenv("RUN_INTEGRATION_TESTS") == "" { t.Skip("skipping integration test; set RUN_INTEGRATION_TESTS=1 to run") } - _, err := getWindows11Download("amd64", filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/windows11/iso/win11_25h2_english_amd64.iso"), false) + _, err := getWindows11Download("amd64", GetDirectoriesInstance().CachePath("windows11", "iso", "win11_25h2_english_amd64.iso"), false) if err != nil { t.Fatalf("Failed to get Windows 11 download: %v", err) } @@ -102,7 +102,7 @@ func TestGetWindows11DownloadArm64(t *testing.T) { if os.Getenv("RUN_INTEGRATION_TESTS") == "" { t.Skip("skipping integration test; set RUN_INTEGRATION_TESTS=1 to run") } - _, err := getWindows11Download("arm64", filepath.Join(GetDirectoriesInstance().ProjectDir, "./cache/windows11/iso/win11_25h2_english_arm64.iso"), false) + _, err := getWindows11Download("arm64", GetDirectoriesInstance().CachePath("windows11", "iso", "win11_25h2_english_arm64.iso"), false) if err != nil { t.Fatalf("Failed to get Windows 11 download: %v", err) } diff --git a/pkg/build/directories.go b/pkg/build/directories.go index 21811a90..ecc75d18 100644 --- a/pkg/build/directories.go +++ b/pkg/build/directories.go @@ -1,15 +1,28 @@ package build import ( + "fmt" "log" "os" "path/filepath" + "runtime" +) + +const ( + devAlchemyAppName = "dev-alchemy" + devAlchemyAppDataEnvVar = "DEV_ALCHEMY_APP_DATA_DIR" + devAlchemyCacheEnvVar = "DEV_ALCHEMY_CACHE_DIR" + devAlchemyVagrantEnvVar = "DEV_ALCHEMY_VAGRANT_DIR" + devAlchemyPackerCacheEnvVar = "DEV_ALCHEMY_PACKER_CACHE_DIR" ) type Directories struct { - WorkingDir string - ProjectDir string - CacheDir string + WorkingDir string + ProjectDir string + AppDataDir string + CacheDir string + VagrantDir string + PackerCacheDir string } var ( @@ -35,8 +48,24 @@ func (u *Directories) GetDirectories() Directories { log.Fatalf("Project dir could not be determined.") } } + if u.AppDataDir == "" { + appDataDir, err := resolveDefaultAppDataDir() + if err != nil { + log.Fatalf("App data dir could not be determined: %v", err) + } + u.AppDataDir = appDataDir + } if u.CacheDir == "" { - u.CacheDir = filepath.Join(u.ProjectDir, "cache") + u.CacheDir = filepath.Join(u.AppDataDir, "cache") + } + if u.VagrantDir == "" { + u.VagrantDir = filepath.Join(u.AppDataDir, ".vagrant") + } + if u.PackerCacheDir == "" { + u.PackerCacheDir = filepath.Join(u.AppDataDir, "packer_cache") + } + if err := ensureDirectoriesExist(u.AppDataDir, u.CacheDir, u.VagrantDir, u.PackerCacheDir); err != nil { + log.Fatalf("Managed application directories could not be created: %v", err) } return *u } @@ -52,8 +81,7 @@ func (u *Directories) determineTopLevelDirWithGit() string { } for { gitPath := filepath.Join(dir, ".git") - info, err := os.Stat(gitPath) - if err == nil && info.IsDir() { + if _, err := os.Stat(gitPath); err == nil { u.ProjectDir = dir return dir } @@ -77,3 +105,86 @@ func (u *Directories) getWorkingDir() string { u.WorkingDir = dir return dir } + +func (u *Directories) CachePath(paths ...string) string { + u.GetDirectories() + return filepath.Join(append([]string{u.CacheDir}, paths...)...) +} + +func (u *Directories) VagrantPath(paths ...string) string { + u.GetDirectories() + return filepath.Join(append([]string{u.VagrantDir}, paths...)...) +} + +func (u *Directories) PackerCachePath(paths ...string) string { + u.GetDirectories() + return filepath.Join(append([]string{u.PackerCacheDir}, paths...)...) +} + +func (u *Directories) ManagedEnv() []string { + u.GetDirectories() + return []string{ + devAlchemyAppDataEnvVar + "=" + u.AppDataDir, + devAlchemyCacheEnvVar + "=" + u.CacheDir, + devAlchemyVagrantEnvVar + "=" + u.VagrantDir, + devAlchemyPackerCacheEnvVar + "=" + u.PackerCacheDir, + "PACKER_CACHE_DIR=" + u.PackerCacheDir, + } +} + +func resolveDefaultAppDataDir() (string, error) { + return resolveDefaultAppDataDirForOS(runtime.GOOS, os.Getenv, os.UserHomeDir, os.UserConfigDir) +} + +func resolveDefaultAppDataDirForOS( + goos string, + getenv func(string) string, + userHomeDir func() (string, error), + userConfigDir func() (string, error), +) (string, error) { + if override := getenv(devAlchemyAppDataEnvVar); override != "" { + return filepath.Clean(override), nil + } + + switch goos { + case "darwin": + homeDir, err := userHomeDir() + if err != nil { + return "", fmt.Errorf("resolve macOS home directory: %w", err) + } + return filepath.Join(homeDir, "Library", "Application Support", devAlchemyAppName), nil + case "windows": + if localAppData := getenv("LOCALAPPDATA"); localAppData != "" { + return filepath.Join(localAppData, devAlchemyAppName), nil + } + if appData := getenv("APPDATA"); appData != "" { + return filepath.Join(appData, devAlchemyAppName), nil + } + configDir, err := userConfigDir() + if err != nil { + return "", fmt.Errorf("resolve Windows app data directory: %w", err) + } + return filepath.Join(configDir, devAlchemyAppName), nil + default: + if xdgDataHome := getenv("XDG_DATA_HOME"); xdgDataHome != "" { + return filepath.Join(xdgDataHome, devAlchemyAppName), nil + } + homeDir, err := userHomeDir() + if err != nil { + return "", fmt.Errorf("resolve Linux home directory: %w", err) + } + return filepath.Join(homeDir, ".local", "share", devAlchemyAppName), nil + } +} + +func ensureDirectoriesExist(paths ...string) error { + for _, currentPath := range paths { + if currentPath == "" { + continue + } + if err := os.MkdirAll(currentPath, 0755); err != nil { + return err + } + } + return nil +} diff --git a/pkg/build/directories_test.go b/pkg/build/directories_test.go new file mode 100644 index 00000000..315efd0a --- /dev/null +++ b/pkg/build/directories_test.go @@ -0,0 +1,117 @@ +package build + +import ( + "errors" + "path/filepath" + "testing" +) + +func TestResolveDefaultAppDataDirForOS_UsesOverride(t *testing.T) { + got, err := resolveDefaultAppDataDirForOS( + "linux", + func(key string) string { + if key == devAlchemyAppDataEnvVar { + return "/tmp/dev-alchemy-custom" + } + return "" + }, + func() (string, error) { return "/home/tester", nil }, + func() (string, error) { return "/config", nil }, + ) + if err != nil { + t.Fatalf("resolveDefaultAppDataDirForOS returned error: %v", err) + } + if got != filepath.Clean("/tmp/dev-alchemy-custom") { + t.Fatalf("expected override path, got %q", got) + } +} + +func TestResolveDefaultAppDataDirForOS_Darwin(t *testing.T) { + got, err := resolveDefaultAppDataDirForOS( + "darwin", + func(string) string { return "" }, + func() (string, error) { return "/Users/tester", nil }, + func() (string, error) { return "", nil }, + ) + if err != nil { + t.Fatalf("resolveDefaultAppDataDirForOS returned error: %v", err) + } + want := filepath.Join("/Users/tester", "Library", "Application Support", devAlchemyAppName) + if got != want { + t.Fatalf("expected %q, got %q", want, got) + } +} + +func TestResolveDefaultAppDataDirForOS_WindowsPrefersLocalAppData(t *testing.T) { + got, err := resolveDefaultAppDataDirForOS( + "windows", + func(key string) string { + switch key { + case "LOCALAPPDATA": + return `C:\Users\tester\AppData\Local` + case "APPDATA": + return `C:\Users\tester\AppData\Roaming` + default: + return "" + } + }, + func() (string, error) { return "", nil }, + func() (string, error) { return `C:\Users\tester\AppData\Roaming`, nil }, + ) + if err != nil { + t.Fatalf("resolveDefaultAppDataDirForOS returned error: %v", err) + } + want := filepath.Join(`C:\Users\tester\AppData\Local`, devAlchemyAppName) + if got != want { + t.Fatalf("expected %q, got %q", want, got) + } +} + +func TestResolveDefaultAppDataDirForOS_LinuxUsesXDGDataHome(t *testing.T) { + got, err := resolveDefaultAppDataDirForOS( + "linux", + func(key string) string { + if key == "XDG_DATA_HOME" { + return "/home/tester/.local/share" + } + return "" + }, + func() (string, error) { return "/home/tester", nil }, + func() (string, error) { return "", nil }, + ) + if err != nil { + t.Fatalf("resolveDefaultAppDataDirForOS returned error: %v", err) + } + want := filepath.Join("/home/tester/.local/share", devAlchemyAppName) + if got != want { + t.Fatalf("expected %q, got %q", want, got) + } +} + +func TestResolveDefaultAppDataDirForOS_LinuxFallsBackToHome(t *testing.T) { + got, err := resolveDefaultAppDataDirForOS( + "linux", + func(string) string { return "" }, + func() (string, error) { return "/home/tester", nil }, + func() (string, error) { return "", nil }, + ) + if err != nil { + t.Fatalf("resolveDefaultAppDataDirForOS returned error: %v", err) + } + want := filepath.Join("/home/tester", ".local", "share", devAlchemyAppName) + if got != want { + t.Fatalf("expected %q, got %q", want, got) + } +} + +func TestResolveDefaultAppDataDirForOS_ReturnsErrorWhenHomeUnavailable(t *testing.T) { + _, err := resolveDefaultAppDataDirForOS( + "darwin", + func(string) string { return "" }, + func() (string, error) { return "", errors.New("boom") }, + func() (string, error) { return "", nil }, + ) + if err == nil { + t.Fatal("expected error when home directory lookup fails") + } +} diff --git a/pkg/build/external_process_handler.go b/pkg/build/external_process_handler.go index 265b4812..c20ef5c0 100644 --- a/pkg/build/external_process_handler.go +++ b/pkg/build/external_process_handler.go @@ -16,6 +16,7 @@ import ( type RunProcessConfig struct { ExecutablePath string Args []string + Env []string WorkingDir string Timeout time.Duration DelayBeforeStart time.Duration @@ -61,6 +62,9 @@ func RunExternalProcess(config RunProcessConfig) (context.Context, error) { // #nosec G204 -- callers pass explicit executables and argv slices; no shell parsing occurs here. cmd := exec.CommandContext(ctx, config.ExecutablePath, config.Args...) cmd.Dir = config.WorkingDir + if len(config.Env) > 0 { + cmd.Env = append(os.Environ(), config.Env...) + } stdout, err := cmd.StdoutPipe() if err != nil { diff --git a/pkg/build/generic_build.go b/pkg/build/generic_build.go index 31e5e4d7..584b6f90 100644 --- a/pkg/build/generic_build.go +++ b/pkg/build/generic_build.go @@ -48,6 +48,7 @@ func RunBuildScript(config VirtualMachineConfig, executable string, args []strin // #nosec G204 -- executable and args are constructed by internal build flows; no shell is invoked. cmd := exec.CommandContext(ctx, executable, args...) cmd.Dir = GetDirectoriesInstance().GetDirectories().ProjectDir + cmd.Env = append(os.Environ(), GetDirectoriesInstance().ManagedEnv()...) readAndPrintStdoutStderr(cmd, config) diff --git a/pkg/build/macos-silicon-build-helper_test.go b/pkg/build/macos-silicon-build-helper_test.go index 9c6d6684..80da19f4 100644 --- a/pkg/build/macos-silicon-build-helper_test.go +++ b/pkg/build/macos-silicon-build-helper_test.go @@ -19,9 +19,14 @@ import ( func TestMacOsDownloadArm64Uefi(t *testing.T) { requireIntegrationTests(t) t.Parallel() + appDataDir := t.TempDir() + cacheDir := filepath.Join(appDataDir, "cache") + t.Setenv(devAlchemyAppDataEnvVar, appDataDir) + t.Setenv(devAlchemyCacheEnvVar, cacheDir) + t.Setenv(devAlchemyPackerCacheEnvVar, filepath.Join(appDataDir, "packer_cache")) // Remove files matching cache/qemu-efi* - matches, err := filepath.Glob("../../cache/qemu-efi*") + matches, err := filepath.Glob(filepath.Join(cacheDir, "qemu-efi*")) if err != nil { t.Fatalf("Failed to glob ../../cache/qemu-efi*: %v", err) } @@ -32,7 +37,7 @@ func TestMacOsDownloadArm64Uefi(t *testing.T) { } // Remove folders matching cache/qemu-uefi - matches, err = filepath.Glob("../../cache/qemu-uefi") + matches, err = filepath.Glob(filepath.Join(cacheDir, "qemu-uefi")) if err != nil { t.Fatalf("Failed to glob ../../cache/qemu-uefi: %v", err) } @@ -54,11 +59,11 @@ func TestMacOsDownloadArm64Uefi(t *testing.T) { t.Fatalf("Failed to run %s: %v", scriptPath, err) } - if _, err := os.Stat("../../cache/qemu-uefi/usr/share/qemu-efi-aarch64/QEMU_EFI.fd"); err != nil { + if _, err := os.Stat(filepath.Join(cacheDir, "qemu-uefi", "usr", "share", "qemu-efi-aarch64", "QEMU_EFI.fd")); err != nil { if os.IsNotExist(err) { - t.Fatalf("Expected file ../../cache/qemu-uefi/usr/share/qemu-efi-aarch64/QEMU_EFI.fd to exist, but it does not") + t.Fatalf("Expected qemu-uefi firmware file to exist in %s, but it does not", cacheDir) } else { - t.Fatalf("Failed to stat ../../cache/qemu-uefi/usr/share/qemu-efi-aarch64/QEMU_EFI.fd: %v", err) + t.Fatalf("Failed to stat qemu-uefi firmware file in %s: %v", cacheDir, err) } } } diff --git a/pkg/build/windows-build.go b/pkg/build/windows-build.go index 4ea892ec..5eac225f 100644 --- a/pkg/build/windows-build.go +++ b/pkg/build/windows-build.go @@ -8,7 +8,6 @@ import ( const ( packerExecutable = "packer" - windows11ISOPath = "./cache/windows11/iso/win11_25h2_english_amd64.iso" hypervPackerFile = "build/packer/windows/windows11-on-windows-hyperv.pkr.hcl" virtualBoxPackerFile = "build/packer/windows/windows11-on-windows-virtualbox.pkr.hcl" ) @@ -90,7 +89,8 @@ func buildPackerArgs(config VirtualMachineConfig, packerFile string) []string { // Add ISO URL, CPU count, memory, and Packer file args = append(args, - "-var", fmt.Sprintf("iso_url=%s", windows11ISOPath), + "-var", fmt.Sprintf("iso_url=%s", GetDirectoriesInstance().CachePath("windows11", "iso", "win11_25h2_english_amd64.iso")), + "-var", fmt.Sprintf("cache_dir=%s", GetDirectoriesInstance().CacheDir), "-var", fmt.Sprintf("cpus=%s", getVmCpuCountString(config)), "-var", fmt.Sprintf("memory=%d", getVmMemoryMB(config)), packerFile, diff --git a/pkg/build/windows-linux-build.go b/pkg/build/windows-linux-build.go index 31817de8..d1789e50 100644 --- a/pkg/build/windows-linux-build.go +++ b/pkg/build/windows-linux-build.go @@ -6,7 +6,6 @@ import ( const ( ubuntuHypervPackerFile = "build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl" - ubuntuServerISOPath = "./cache/linux/ubuntu-24.04.3-live-server-amd64.iso" ) // RunHypervUbuntuBuildOnWindows builds an Ubuntu VM using Hyper-V on Windows. @@ -17,7 +16,8 @@ func RunHypervUbuntuBuildOnWindows(config VirtualMachineConfig) error { args := []string{ "build", - "-var", fmt.Sprintf("iso_url=%s", ubuntuServerISOPath), + "-var", fmt.Sprintf("iso_url=%s", GetDirectoriesInstance().CachePath("linux", "ubuntu-24.04.3-live-server-amd64.iso")), + "-var", fmt.Sprintf("cache_dir=%s", GetDirectoriesInstance().CacheDir), "-var", fmt.Sprintf("ubuntu_type=%s", defaultUbuntuType(config.UbuntuType)), "-var", fmt.Sprintf("cpus=%s", getVmCpuCountString(config)), "-var", fmt.Sprintf("memory=%d", getVmMemoryMB(config)), diff --git a/pkg/deploy/windows-hyperv-deploy.go b/pkg/deploy/windows-hyperv-deploy.go index e43c9ac9..41c44d5d 100644 --- a/pkg/deploy/windows-hyperv-deploy.go +++ b/pkg/deploy/windows-hyperv-deploy.go @@ -2,7 +2,6 @@ package deploy import ( "fmt" - "path" "path/filepath" "strconv" "strings" @@ -324,7 +323,7 @@ func ResolveHypervVagrantExecutionSettings(config alchemy_build.VirtualMachineCo } func hypervVagrantDotfilePath(vmName string) string { - return path.Join(".vagrant", vmName) + return alchemy_build.GetDirectoriesInstance().VagrantPath(vmName) } func buildHypervVagrantResourceEnv(config alchemy_build.VirtualMachineConfig) []string { diff --git a/pkg/deploy/windows-hyperv-deploy_settings_test.go b/pkg/deploy/windows-hyperv-deploy_settings_test.go index 3c882704..3faceea3 100644 --- a/pkg/deploy/windows-hyperv-deploy_settings_test.go +++ b/pkg/deploy/windows-hyperv-deploy_settings_test.go @@ -1,7 +1,6 @@ package deploy import ( - "path" "path/filepath" "strconv" "testing" @@ -11,6 +10,7 @@ import ( func TestResolveHypervVagrantDeploySettings_UbuntuIncludesConfigResources(t *testing.T) { projectDir := t.TempDir() + vagrantDir := setDeploySettingsTestVagrantRoot(t) config := alchemy_build.VirtualMachineConfig{ OS: "ubuntu", UbuntuType: "server", @@ -40,7 +40,7 @@ func TestResolveHypervVagrantDeploySettings_UbuntuIncludesConfigResources(t *tes if env[hypervVagrantVMNameEnvVar] != "linux-ubuntu-server-packer" { t.Fatalf("expected %s env var to match box name, got %q", hypervVagrantVMNameEnvVar, env[hypervVagrantVMNameEnvVar]) } - expectedDotfilePath := path.Join(".vagrant", "linux-ubuntu-server-packer") + expectedDotfilePath := filepath.Join(vagrantDir, "linux-ubuntu-server-packer") if env[hypervVagrantDotfileEnvVar] != expectedDotfilePath { t.Fatalf("expected %s env var to match isolated state path, got %q", hypervVagrantDotfileEnvVar, env[hypervVagrantDotfileEnvVar]) } @@ -59,6 +59,7 @@ func TestResolveHypervVagrantDeploySettings_UbuntuIncludesConfigResources(t *tes func TestResolveHypervVagrantDeploySettings_WindowsIncludesConfigResources(t *testing.T) { projectDir := t.TempDir() + vagrantDir := setDeploySettingsTestVagrantRoot(t) config := alchemy_build.VirtualMachineConfig{ OS: "windows11", Arch: "amd64", @@ -87,7 +88,7 @@ func TestResolveHypervVagrantDeploySettings_WindowsIncludesConfigResources(t *te if env[hypervVagrantVMNameEnvVar] != windowsHypervVagrantBoxName { t.Fatalf("expected %s env var to match windows vm name, got %q", hypervVagrantVMNameEnvVar, env[hypervVagrantVMNameEnvVar]) } - expectedDotfilePath := path.Join(".vagrant", windowsHypervVagrantBoxName) + expectedDotfilePath := filepath.Join(vagrantDir, windowsHypervVagrantBoxName) if env[hypervVagrantDotfileEnvVar] != expectedDotfilePath { t.Fatalf("expected %s env var to match isolated state path, got %q", hypervVagrantDotfileEnvVar, env[hypervVagrantDotfileEnvVar]) } @@ -106,6 +107,7 @@ func TestResolveHypervVagrantDeploySettings_WindowsIncludesConfigResources(t *te func TestResolveHypervVagrantDeploySettings_UbuntuVariantsUseDistinctDotfilePaths(t *testing.T) { projectDir := t.TempDir() + _ = setDeploySettingsTestVagrantRoot(t) serverSettings, err := resolveHypervVagrantDeploySettings(alchemy_build.VirtualMachineConfig{ OS: "ubuntu", @@ -144,3 +146,21 @@ func envListToMap(envList []string) map[string]string { } return env } + +func setDeploySettingsTestVagrantRoot(t *testing.T) string { + t.Helper() + + dirs := alchemy_build.GetDirectoriesInstance() + originalAppDataDir := dirs.AppDataDir + originalVagrantDir := dirs.VagrantDir + appDataDir := t.TempDir() + vagrantDir := filepath.Join(appDataDir, ".vagrant") + dirs.AppDataDir = appDataDir + dirs.VagrantDir = vagrantDir + t.Cleanup(func() { + dirs.AppDataDir = originalAppDataDir + dirs.VagrantDir = originalVagrantDir + }) + + return vagrantDir +} diff --git a/pkg/deploy/windows-hyperv-destroy_test.go b/pkg/deploy/windows-hyperv-destroy_test.go index bcb3943b..1fc92629 100644 --- a/pkg/deploy/windows-hyperv-destroy_test.go +++ b/pkg/deploy/windows-hyperv-destroy_test.go @@ -2,6 +2,7 @@ package deploy import ( "fmt" + "path/filepath" "testing" "time" @@ -118,6 +119,7 @@ func TestHypervStartTargetStateFromVMState(t *testing.T) { func TestRunHypervVagrantStopOnWindows_TreatsGracefulHaltErrorAsSuccessOnceStopped(t *testing.T) { restore := stubHypervStopDependencies(t) defer restore() + vagrantRoot := setHypervTestVagrantRoot(t) commands := make([][]string, 0, 1) runHypervVagrantCommandWithEnv = func(_ string, _ time.Duration, executable string, args []string, env []string, _ string) error { @@ -126,7 +128,7 @@ func TestRunHypervVagrantStopOnWindows_TreatsGracefulHaltErrorAsSuccessOnceStopp } assertEnvContainsEntry(t, env, "VAGRANT_BOX_NAME=linux-ubuntu-server-packer") assertEnvContainsEntry(t, env, "VAGRANT_VM_NAME=linux-ubuntu-server-packer") - assertEnvContainsEntry(t, env, "VAGRANT_DOTFILE_PATH=.vagrant/linux-ubuntu-server-packer") + assertEnvContainsEntry(t, env, "VAGRANT_DOTFILE_PATH="+filepath.Join(vagrantRoot, "linux-ubuntu-server-packer")) commands = append(commands, append([]string(nil), args...)) return fmt.Errorf("command failed (vagrant [halt]): exit status 1") } @@ -167,6 +169,7 @@ func TestRunHypervVagrantStopOnWindows_TreatsGracefulHaltErrorAsSuccessOnceStopp func TestRunHypervVagrantStopOnWindows_FallsBackToForcedHalt(t *testing.T) { restore := stubHypervStopDependencies(t) defer restore() + vagrantRoot := setHypervTestVagrantRoot(t) commands := make([][]string, 0, 2) runHypervVagrantCommandWithEnv = func(_ string, _ time.Duration, executable string, args []string, env []string, _ string) error { @@ -175,7 +178,7 @@ func TestRunHypervVagrantStopOnWindows_FallsBackToForcedHalt(t *testing.T) { } assertEnvContainsEntry(t, env, "VAGRANT_BOX_NAME=linux-ubuntu-desktop-packer") assertEnvContainsEntry(t, env, "VAGRANT_VM_NAME=linux-ubuntu-desktop-packer") - assertEnvContainsEntry(t, env, "VAGRANT_DOTFILE_PATH=.vagrant/linux-ubuntu-desktop-packer") + assertEnvContainsEntry(t, env, "VAGRANT_DOTFILE_PATH="+filepath.Join(vagrantRoot, "linux-ubuntu-desktop-packer")) commands = append(commands, append([]string(nil), args...)) if len(args) == 1 && args[0] == "halt" { return fmt.Errorf("command failed (vagrant [halt]): exit status 1") @@ -257,6 +260,7 @@ func TestRunHypervVagrantStopOnWindows_ReturnsErrorWhenVMStillRunningAfterForced func TestRunHypervVagrantStartOnWindows_UsesTypeSpecificVagrantEnv(t *testing.T) { restore := stubHypervStopDependencies(t) defer restore() + vagrantRoot := setHypervTestVagrantRoot(t) inspectHypervVagrantStartCmdTarget = func(alchemy_build.VirtualMachineConfig) (StartTargetState, error) { return StartTargetState{Exists: true, State: "off"}, nil @@ -276,7 +280,7 @@ func TestRunHypervVagrantStartOnWindows_UsesTypeSpecificVagrantEnv(t *testing.T) } assertEnvContainsEntry(t, env, "VAGRANT_BOX_NAME=linux-ubuntu-desktop-packer") assertEnvContainsEntry(t, env, "VAGRANT_VM_NAME=linux-ubuntu-desktop-packer") - assertEnvContainsEntry(t, env, "VAGRANT_DOTFILE_PATH=.vagrant/linux-ubuntu-desktop-packer") + assertEnvContainsEntry(t, env, "VAGRANT_DOTFILE_PATH="+filepath.Join(vagrantRoot, "linux-ubuntu-desktop-packer")) return nil } @@ -298,6 +302,7 @@ func TestRunHypervVagrantStartOnWindows_UsesTypeSpecificVagrantEnv(t *testing.T) func TestRunHypervVagrantDestroyOnWindows_UsesTypeSpecificVagrantEnv(t *testing.T) { restore := stubHypervStopDependencies(t) defer restore() + vagrantRoot := setHypervTestVagrantRoot(t) hypervVagrantMachineExistsChecker = func(workingDir string, env []string) (bool, error) { if workingDir == "" { @@ -305,7 +310,7 @@ func TestRunHypervVagrantDestroyOnWindows_UsesTypeSpecificVagrantEnv(t *testing. } assertEnvContainsEntry(t, env, "VAGRANT_BOX_NAME=linux-ubuntu-desktop-packer") assertEnvContainsEntry(t, env, "VAGRANT_VM_NAME=linux-ubuntu-desktop-packer") - assertEnvContainsEntry(t, env, "VAGRANT_DOTFILE_PATH=.vagrant/linux-ubuntu-desktop-packer") + assertEnvContainsEntry(t, env, "VAGRANT_DOTFILE_PATH="+filepath.Join(vagrantRoot, "linux-ubuntu-desktop-packer")) return true, nil } hypervVagrantBoxInstalledChecker = func(projectDir string, boxName string) (bool, error) { @@ -332,7 +337,7 @@ func TestRunHypervVagrantDestroyOnWindows_UsesTypeSpecificVagrantEnv(t *testing. } assertEnvContainsEntry(t, env, "VAGRANT_BOX_NAME=linux-ubuntu-desktop-packer") assertEnvContainsEntry(t, env, "VAGRANT_VM_NAME=linux-ubuntu-desktop-packer") - assertEnvContainsEntry(t, env, "VAGRANT_DOTFILE_PATH=.vagrant/linux-ubuntu-desktop-packer") + assertEnvContainsEntry(t, env, "VAGRANT_DOTFILE_PATH="+filepath.Join(vagrantRoot, "linux-ubuntu-desktop-packer")) return nil } @@ -354,6 +359,7 @@ func TestRunHypervVagrantDestroyOnWindows_UsesTypeSpecificVagrantEnv(t *testing. func TestDestroyTargetExists_UsesTypeSpecificVagrantEnv(t *testing.T) { restore := stubHypervStopDependencies(t) defer restore() + vagrantRoot := setHypervTestVagrantRoot(t) hypervVagrantMachineExistsChecker = func(workingDir string, env []string) (bool, error) { if workingDir == "" { @@ -361,7 +367,7 @@ func TestDestroyTargetExists_UsesTypeSpecificVagrantEnv(t *testing.T) { } assertEnvContainsEntry(t, env, "VAGRANT_BOX_NAME=linux-ubuntu-desktop-packer") assertEnvContainsEntry(t, env, "VAGRANT_VM_NAME=linux-ubuntu-desktop-packer") - assertEnvContainsEntry(t, env, "VAGRANT_DOTFILE_PATH=.vagrant/linux-ubuntu-desktop-packer") + assertEnvContainsEntry(t, env, "VAGRANT_DOTFILE_PATH="+filepath.Join(vagrantRoot, "linux-ubuntu-desktop-packer")) return false, nil } hypervVagrantBoxInstalledChecker = func(_ string, boxName string) (bool, error) { @@ -411,6 +417,24 @@ func stubHypervStopDependencies(t *testing.T) func() { } } +func setHypervTestVagrantRoot(t *testing.T) string { + t.Helper() + + dirs := alchemy_build.GetDirectoriesInstance() + originalAppDataDir := dirs.AppDataDir + originalVagrantDir := dirs.VagrantDir + appDataDir := t.TempDir() + vagrantDir := filepath.Join(appDataDir, ".vagrant") + dirs.AppDataDir = appDataDir + dirs.VagrantDir = vagrantDir + t.Cleanup(func() { + dirs.AppDataDir = originalAppDataDir + dirs.VagrantDir = originalVagrantDir + }) + + return vagrantDir +} + func TestVagrantBoxListIncludesMatchesExactNameAndProvider(t *testing.T) { output := "win11-packer (hyperv, 0)\nlinux-ubuntu-server-packer (hyperv, 0)\n" diff --git a/pkg/deploy/windows-hyperv-settings_test.go b/pkg/deploy/windows-hyperv-settings_test.go index a7699b5b..7f1e8d72 100644 --- a/pkg/deploy/windows-hyperv-settings_test.go +++ b/pkg/deploy/windows-hyperv-settings_test.go @@ -1,7 +1,6 @@ package deploy import ( - "path" "path/filepath" "testing" @@ -11,9 +10,16 @@ import ( func TestResolveHypervVagrantExecutionSettings_UsesUbuntuTypeSpecificMetadata(t *testing.T) { dirs := alchemy_build.GetDirectoriesInstance() originalProjectDir := dirs.ProjectDir + originalAppDataDir := dirs.AppDataDir + originalVagrantDir := dirs.VagrantDir + appDataDir := t.TempDir() dirs.ProjectDir = t.TempDir() + dirs.AppDataDir = appDataDir + dirs.VagrantDir = filepath.Join(appDataDir, ".vagrant") t.Cleanup(func() { dirs.ProjectDir = originalProjectDir + dirs.AppDataDir = originalAppDataDir + dirs.VagrantDir = originalVagrantDir }) settings, err := ResolveHypervVagrantExecutionSettings(alchemy_build.VirtualMachineConfig{ @@ -32,7 +38,7 @@ func TestResolveHypervVagrantExecutionSettings_UsesUbuntuTypeSpecificMetadata(t assertEnvContains(t, settings.VagrantEnv, "VAGRANT_BOX_NAME=linux-ubuntu-desktop-packer") assertEnvContains(t, settings.VagrantEnv, "VAGRANT_VM_NAME=linux-ubuntu-desktop-packer") - assertEnvContains(t, settings.VagrantEnv, "VAGRANT_DOTFILE_PATH="+path.Join(".vagrant", "linux-ubuntu-desktop-packer")) + assertEnvContains(t, settings.VagrantEnv, "VAGRANT_DOTFILE_PATH="+filepath.Join(dirs.VagrantDir, "linux-ubuntu-desktop-packer")) } func assertEnvContains(t *testing.T, env []string, want string) { diff --git a/scripts/macos/README.md b/scripts/macos/README.md index a06ab839..51fdee10 100644 --- a/scripts/macos/README.md +++ b/scripts/macos/README.md @@ -38,16 +38,25 @@ Or run the script with the `--headless false` flag to see the browser in action: python playwright_win11_iso.py --headless false ``` This will output the latest Windows 11 ISO download link in the terminal. -Additionally , the script saves the download link to a file named `./cache/windows/win11_amd64_iso_url.txt` or `./cache/windows/win11_arm64_iso_url.txt`. +Additionally, the script saves the download link to the managed cache directory: + +- macOS default: `~/Library/Application Support/dev-alchemy/cache/windows/win11_amd64_iso_url.txt` +- macOS default: `~/Library/Application Support/dev-alchemy/cache/windows/win11_arm64_iso_url.txt` + +Set `DEV_ALCHEMY_APP_DATA_DIR` to move the managed cache elsewhere. ### Download the ISO You can use `curl` or `wget` to download the ISO using the link saved in the file: ```bash -cd ./cache/windows/ -curl --progress-bar -o win11_25h2_english_x64.iso $(cat ./win11_iso_url.txt) +APP_DATA_DIR="${DEV_ALCHEMY_APP_DATA_DIR:-$HOME/Library/Application Support/dev-alchemy}" +CACHE_DIR="${DEV_ALCHEMY_CACHE_DIR:-$APP_DATA_DIR/cache}" +cd "$CACHE_DIR/windows/" +curl --progress-bar -o ../windows11/iso/win11_25h2_english_amd64.iso "$(cat ./win11_amd64_iso_url.txt)" ``` or for arm: ```bash -cd ./cache/windows/ -curl --progress-bar -o win11_25h2_english_arm64.iso $(cat ./win11_arm_iso_url.txt) -``` \ No newline at end of file +APP_DATA_DIR="${DEV_ALCHEMY_APP_DATA_DIR:-$HOME/Library/Application Support/dev-alchemy}" +CACHE_DIR="${DEV_ALCHEMY_CACHE_DIR:-$APP_DATA_DIR/cache}" +cd "$CACHE_DIR/windows/" +curl --progress-bar -o ../windows11/iso/win11_25h2_english_arm64.iso "$(cat ./win11_arm64_iso_url.txt)" +``` diff --git a/scripts/macos/create-qemu-qcow2-disk.sh b/scripts/macos/create-qemu-qcow2-disk.sh index 4a2f057b..bf8da7a3 100644 --- a/scripts/macos/create-qemu-qcow2-disk.sh +++ b/scripts/macos/create-qemu-qcow2-disk.sh @@ -28,7 +28,10 @@ script_dir=$( pwd ) -cache_dir="$script_dir/../../cache" +app_data_dir="${DEV_ALCHEMY_APP_DATA_DIR:-$HOME/Library/Application Support/dev-alchemy}" +cache_dir="${DEV_ALCHEMY_CACHE_DIR:-$app_data_dir/cache}" +export DEV_ALCHEMY_APP_DATA_DIR="$app_data_dir" +export DEV_ALCHEMY_CACHE_DIR="$cache_dir" mkdir -p "$cache_dir/windows11/" rm -f "$cache_dir/windows11/qemu-windows11-$arch.qcow2" diff --git a/scripts/macos/create-win11-autounattend-iso.sh b/scripts/macos/create-win11-autounattend-iso.sh index cccf1079..c9c0e003 100644 --- a/scripts/macos/create-win11-autounattend-iso.sh +++ b/scripts/macos/create-win11-autounattend-iso.sh @@ -7,8 +7,13 @@ SCRIPT_DIR=$( pwd ) -vendor_dir="$SCRIPT_DIR/../../cache/windows" -iso_dir="$SCRIPT_DIR/../../cache/windows11/iso" +app_data_dir="${DEV_ALCHEMY_APP_DATA_DIR:-$HOME/Library/Application Support/dev-alchemy}" +cache_dir="${DEV_ALCHEMY_CACHE_DIR:-$app_data_dir/cache}" +export DEV_ALCHEMY_APP_DATA_DIR="$app_data_dir" +export DEV_ALCHEMY_CACHE_DIR="$cache_dir" + +vendor_dir="$cache_dir/windows" +iso_dir="$cache_dir/windows11/iso" autounattend_xml_path="$SCRIPT_DIR/../../build/packer/windows/qemu-arm64/autounattend.xml" windows_source_iso_path="$iso_dir/win11_25h2_english_arm64.iso" windows_target_iso_path="$iso_dir/Win11_ARM64_Unattended.iso" diff --git a/scripts/macos/download-arm64-uefi.sh b/scripts/macos/download-arm64-uefi.sh index 264babad..87cb3db3 100644 --- a/scripts/macos/download-arm64-uefi.sh +++ b/scripts/macos/download-arm64-uefi.sh @@ -7,10 +7,15 @@ SCRIPT_DIR=$( pwd ) -DEB_PATH="${SCRIPT_DIR}/../../cache/qemu-efi-aarch64_all.deb" +app_data_dir="${DEV_ALCHEMY_APP_DATA_DIR:-$HOME/Library/Application Support/dev-alchemy}" +cache_dir="${DEV_ALCHEMY_CACHE_DIR:-$app_data_dir/cache}" +export DEV_ALCHEMY_APP_DATA_DIR="$app_data_dir" +export DEV_ALCHEMY_CACHE_DIR="$cache_dir" + +DEB_PATH="${cache_dir}/qemu-efi-aarch64_all.deb" # Ensure cache directory exists before downloading -mkdir -p "${SCRIPT_DIR}"/../../cache +mkdir -p "${cache_dir}" if [ ! -f "${DEB_PATH}" ]; then echo "Resolving latest qemu-efi-aarch64 download URL from Debian trixie package index" @@ -27,16 +32,16 @@ else echo "qemu-efi-aarch64_all.deb already exists, skipping download" fi -mkdir -p "${SCRIPT_DIR}"/../../cache/qemu-uefi -if [ ! -f "${SCRIPT_DIR}"/../../cache/qemu-uefi/data.tar.xz ]; then +mkdir -p "${cache_dir}/qemu-uefi" +if [ ! -f "${cache_dir}/qemu-uefi/data.tar.xz" ]; then echo "Extract qemu-uefi data.tar.xz" - tar -xvf "${DEB_PATH}" -C "${SCRIPT_DIR}"/../../cache/qemu-uefi + tar -xvf "${DEB_PATH}" -C "${cache_dir}/qemu-uefi" else echo "qemu-uefi/data.tar.xz already exists, skipping extraction" fi -if [ ! -d "${SCRIPT_DIR}"/../../cache/qemu-uefi/usr/share/qemu-efi-aarch64 ]; then - tar -xvf "${SCRIPT_DIR}"/../../cache/qemu-uefi/data.tar.xz -C "${SCRIPT_DIR}"/../../cache/qemu-uefi +if [ ! -d "${cache_dir}/qemu-uefi/usr/share/qemu-efi-aarch64" ]; then + tar -xvf "${cache_dir}/qemu-uefi/data.tar.xz" -C "${cache_dir}/qemu-uefi" else echo "qemu-uefi/usr/share/qemu-efi-aarch64 already exists, skipping extraction" fi diff --git a/scripts/macos/download-utm-guest-tools.sh b/scripts/macos/download-utm-guest-tools.sh index f374bcd2..7fd2264b 100644 --- a/scripts/macos/download-utm-guest-tools.sh +++ b/scripts/macos/download-utm-guest-tools.sh @@ -4,7 +4,11 @@ # and saves it to the cache/utm directory. set -ex SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -OUTPUT_DIR="$SCRIPT_DIR/../../cache/utm" +APP_DATA_DIR="${DEV_ALCHEMY_APP_DATA_DIR:-$HOME/Library/Application Support/dev-alchemy}" +CACHE_DIR="${DEV_ALCHEMY_CACHE_DIR:-$APP_DATA_DIR/cache}" +export DEV_ALCHEMY_APP_DATA_DIR="$APP_DATA_DIR" +export DEV_ALCHEMY_CACHE_DIR="$CACHE_DIR" +OUTPUT_DIR="$CACHE_DIR/utm" OUTPUT_PATH="$OUTPUT_DIR/utm-guest-tools-latest.iso" if [ -f "$OUTPUT_PATH" ]; then diff --git a/scripts/macos/download-virtio-win-iso.sh b/scripts/macos/download-virtio-win-iso.sh index 93d90b5e..e4f7a371 100644 --- a/scripts/macos/download-virtio-win-iso.sh +++ b/scripts/macos/download-virtio-win-iso.sh @@ -7,10 +7,15 @@ SCRIPT_DIR=$( pwd ) -if [ ! -f $SCRIPT_DIR/../../cache/windows/virtio-win.iso ]; then +APP_DATA_DIR="${DEV_ALCHEMY_APP_DATA_DIR:-$HOME/Library/Application Support/dev-alchemy}" +CACHE_DIR="${DEV_ALCHEMY_CACHE_DIR:-$APP_DATA_DIR/cache}" +export DEV_ALCHEMY_APP_DATA_DIR="$APP_DATA_DIR" +export DEV_ALCHEMY_CACHE_DIR="$CACHE_DIR" + +if [ ! -f "$CACHE_DIR/windows/virtio-win.iso" ]; then echo "Downloading virtio-win.iso" - mkdir -p $SCRIPT_DIR/../../cache/windows - curl --progress-bar -L -o $SCRIPT_DIR/../../cache/windows/virtio-win.iso https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/archive-virtio/virtio-win-0.1.266-1/virtio-win-0.1.266.iso + mkdir -p "$CACHE_DIR/windows" + curl --progress-bar -L -o "$CACHE_DIR/windows/virtio-win.iso" https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/archive-virtio/virtio-win-0.1.266-1/virtio-win-0.1.266.iso else echo "virtio-win.iso already exists, skipping download" exit 0 diff --git a/scripts/macos/playwright_win11_iso.py b/scripts/macos/playwright_win11_iso.py index 0429c9af..7c97ff3d 100644 --- a/scripts/macos/playwright_win11_iso.py +++ b/scripts/macos/playwright_win11_iso.py @@ -11,6 +11,7 @@ import argparse import os import json +import sys MICROSOFT_WIN11_ISO_URL = "https://www.microsoft.com/en-US/software-download/windows11" MICROSOFT_WIN11_ARM_ISO_URL = ( @@ -35,6 +36,39 @@ ] +def resolve_app_data_dir() -> str: + override = os.environ.get("DEV_ALCHEMY_APP_DATA_DIR") + if override: + return os.path.abspath(override) + + if sys.platform == "darwin": + return os.path.join( + os.path.expanduser("~"), + "Library", + "Application Support", + "dev-alchemy", + ) + if os.name == "nt": + base = os.environ.get("LOCALAPPDATA") or os.environ.get("APPDATA") + if base: + return os.path.join(base, "dev-alchemy") + return os.path.join( + os.path.expanduser("~"), "AppData", "Local", "dev-alchemy" + ) + + base = os.environ.get("XDG_DATA_HOME") + if base: + return os.path.join(base, "dev-alchemy") + return os.path.join(os.path.expanduser("~"), ".local", "share", "dev-alchemy") + + +def resolve_cache_dir() -> str: + override = os.environ.get("DEV_ALCHEMY_CACHE_DIR") + if override: + return os.path.abspath(override) + return os.path.join(resolve_app_data_dir(), "cache") + + async def random_mouse_movements(page, min_seconds=5, max_seconds=15): duration = random.uniform(min_seconds, max_seconds) start_time = time.time() @@ -195,7 +229,7 @@ async def fetch_win11_iso_link( # Write link to file for use in packer build script_dir = os.path.dirname(os.path.abspath(__file__)) - output_dir = os.path.join(script_dir, "../../cache/windows/") + output_dir = os.path.join(resolve_cache_dir(), "windows") os.makedirs(output_dir, exist_ok=True) output_path = os.path.join( output_dir, "win11_arm64_iso_url.txt" if arm else "win11_amd64_iso_url.txt" diff --git a/scripts/windows/download_win_11.ps1 b/scripts/windows/download_win_11.ps1 index 656686d6..5aacc878 100644 --- a/scripts/windows/download_win_11.ps1 +++ b/scripts/windows/download_win_11.ps1 @@ -5,7 +5,20 @@ param( # Define variables $FidoVersion = "1.67" $FidoExe = "Fido.ps1" -$CacheDir = "$PSScriptRoot\..\..\cache" +$AppDataDir = if ($env:DEV_ALCHEMY_APP_DATA_DIR) { + $env:DEV_ALCHEMY_APP_DATA_DIR +} elseif ($env:LOCALAPPDATA) { + Join-Path $env:LOCALAPPDATA "dev-alchemy" +} elseif ($env:APPDATA) { + Join-Path $env:APPDATA "dev-alchemy" +} else { + Join-Path $HOME "AppData\Local\dev-alchemy" +} +$CacheDir = if ($env:DEV_ALCHEMY_CACHE_DIR) { + $env:DEV_ALCHEMY_CACHE_DIR +} else { + Join-Path $AppDataDir "cache" +} $FidoPath = "$CacheDir\$FidoExe" # LZMA and signature file paths $FidoLzma = "$CacheDir\Fido.ps1.lzma" @@ -74,4 +87,4 @@ if ($IsoPath) { exit 1 } -Set-Location $oldLocation \ No newline at end of file +Set-Location $oldLocation From aa924f426b3f23c29ec46ef31797c53e9a062dde Mon Sep 17 00:00:00 2001 From: Carl-Christian Sautter Date: Sat, 28 Mar 2026 10:48:33 +0000 Subject: [PATCH 02/10] fix(packer): use valid validation messages for cache_dir --- build/packer/linux/mint/linux-mint-hyperv.pkr.hcl | 2 +- build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl | 2 +- build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl | 2 +- build/packer/windows/windows11-on-macos.pkr.hcl | 2 +- build/packer/windows/windows11-on-windows-hyperv.pkr.hcl | 2 +- build/packer/windows/windows11-on-windows-virtualbox.pkr.hcl | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build/packer/linux/mint/linux-mint-hyperv.pkr.hcl b/build/packer/linux/mint/linux-mint-hyperv.pkr.hcl index 27d8db20..c9a3f301 100644 --- a/build/packer/linux/mint/linux-mint-hyperv.pkr.hcl +++ b/build/packer/linux/mint/linux-mint-hyperv.pkr.hcl @@ -28,7 +28,7 @@ variable "cache_dir" { description = "Managed cache directory outside the repository." validation { condition = var.cache_dir != "" - error_message = "cache_dir must be set, typically via DEV_ALCHEMY_CACHE_DIR." + error_message = "The cache_dir variable must be set, typically via DEV_ALCHEMY_CACHE_DIR." } } diff --git a/build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl b/build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl index e9124ed2..47b28596 100644 --- a/build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl +++ b/build/packer/linux/ubuntu/linux-ubuntu-hyperv.pkr.hcl @@ -50,7 +50,7 @@ variable "cache_dir" { description = "Managed cache directory outside the repository." validation { condition = var.cache_dir != "" - error_message = "cache_dir must be set, typically via DEV_ALCHEMY_CACHE_DIR." + error_message = "The cache_dir variable must be set, typically via DEV_ALCHEMY_CACHE_DIR." } } diff --git a/build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl b/build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl index bc9d9e42..97e38521 100644 --- a/build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl +++ b/build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl @@ -70,7 +70,7 @@ variable "cache_dir" { description = "Managed cache directory outside the repository." validation { condition = var.cache_dir != "" - error_message = "cache_dir must be set, typically via DEV_ALCHEMY_CACHE_DIR." + error_message = "The cache_dir variable must be set, typically via DEV_ALCHEMY_CACHE_DIR." } } diff --git a/build/packer/windows/windows11-on-macos.pkr.hcl b/build/packer/windows/windows11-on-macos.pkr.hcl index bfd73229..93fab6c2 100644 --- a/build/packer/windows/windows11-on-macos.pkr.hcl +++ b/build/packer/windows/windows11-on-macos.pkr.hcl @@ -51,7 +51,7 @@ variable "cache_dir" { description = "Managed cache directory outside the repository." validation { condition = var.cache_dir != "" - error_message = "cache_dir must be set, typically via DEV_ALCHEMY_CACHE_DIR." + error_message = "The cache_dir variable must be set, typically via DEV_ALCHEMY_CACHE_DIR." } } diff --git a/build/packer/windows/windows11-on-windows-hyperv.pkr.hcl b/build/packer/windows/windows11-on-windows-hyperv.pkr.hcl index 3d58b8c0..1c5f7dd7 100644 --- a/build/packer/windows/windows11-on-windows-hyperv.pkr.hcl +++ b/build/packer/windows/windows11-on-windows-hyperv.pkr.hcl @@ -40,7 +40,7 @@ variable "cache_dir" { description = "Managed cache directory outside the repository." validation { condition = var.cache_dir != "" - error_message = "cache_dir must be set, typically via DEV_ALCHEMY_CACHE_DIR." + error_message = "The cache_dir variable must be set, typically via DEV_ALCHEMY_CACHE_DIR." } } diff --git a/build/packer/windows/windows11-on-windows-virtualbox.pkr.hcl b/build/packer/windows/windows11-on-windows-virtualbox.pkr.hcl index 4e7fd7b5..369f2729 100644 --- a/build/packer/windows/windows11-on-windows-virtualbox.pkr.hcl +++ b/build/packer/windows/windows11-on-windows-virtualbox.pkr.hcl @@ -46,7 +46,7 @@ variable "cache_dir" { description = "Managed cache directory outside the repository." validation { condition = var.cache_dir != "" - error_message = "cache_dir must be set, typically via DEV_ALCHEMY_CACHE_DIR." + error_message = "The cache_dir variable must be set, typically via DEV_ALCHEMY_CACHE_DIR." } } From 20354ec011cd5d5d394e091e622061f1814971d8 Mon Sep 17 00:00:00 2001 From: Carl-Christian Sautter Date: Sat, 28 Mar 2026 12:31:33 +0000 Subject: [PATCH 03/10] fix(build): shorten macos qemu output paths and stop noisy vnc retries --- .../ubuntu/linux-ubuntu-on-macos.pkr.hcl | 8 +++- .../linux/ubuntu/linux-ubuntu-on-macos.sh | 19 ++++++++- .../packer/windows/windows11-on-macos.pkr.hcl | 8 +++- build/packer/windows/windows11-on-macos.sh | 19 ++++++++- pkg/build/external_process_handler.go | 11 +++++ pkg/build/external_process_handler_test.go | 42 +++++++++++++++++++ pkg/build/helper.go | 4 ++ pkg/build/helper_test.go | 20 +++++++++ pkg/build/macos-silicon-build.go | 38 ++++++++++++++--- 9 files changed, 157 insertions(+), 12 deletions(-) create mode 100644 pkg/build/external_process_handler_test.go create mode 100644 pkg/build/helper_test.go diff --git a/build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl b/build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl index 97e38521..441c0cd8 100644 --- a/build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl +++ b/build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl @@ -74,6 +74,12 @@ variable "cache_dir" { } } +variable "build_output_dir" { + type = string + default = "" + description = "Optional short-lived Packer output directory to avoid long UNIX socket paths on macOS." +} + locals { iso_url = var.iso_url ubuntu_iso_checksum = var.arch == "amd64" ? "sha256:c3514bf0056180d09376462a7a1b4f213c1d6e8ea67fae5c25099c6fd3d8274b" : "none" @@ -132,7 +138,7 @@ locals { ] } left_list = join("", [for i in range(0, 16) : ""]) - output_directory = "${local.cache_directory}/ubuntu/qemu-out-ubuntu-${var.ubuntu_type}-${var.arch}" + output_directory = var.build_output_dir != "" ? var.build_output_dir : "${local.cache_directory}/ubuntu/qemu-out-ubuntu-${var.ubuntu_type}-${var.arch}" } source "qemu" "ubuntu" { diff --git a/build/packer/linux/ubuntu/linux-ubuntu-on-macos.sh b/build/packer/linux/ubuntu/linux-ubuntu-on-macos.sh index e388bc11..955602f3 100644 --- a/build/packer/linux/ubuntu/linux-ubuntu-on-macos.sh +++ b/build/packer/linux/ubuntu/linux-ubuntu-on-macos.sh @@ -11,6 +11,7 @@ vnc_port="5901" cpus="4" memory="4096" verbose="false" +build_output_dir="" script_dir=$( # shellcheck disable=SC2164 @@ -86,6 +87,15 @@ while [[ $# -gt 0 ]]; do exit 1 fi ;; + --build-output-dir) + if [[ -n "$2" ]]; then + build_output_dir="$2" + shift 2 + else + echo "Invalid value for --build-output-dir: $2." >&2 + exit 1 + fi + ;; --verbose) set -x verbose="true" @@ -98,6 +108,10 @@ while [[ $# -gt 0 ]]; do esac done +if [[ -z "$build_output_dir" ]]; then + build_output_dir="/tmp/dev-alchemy/qemu-out-ubuntu-${ubuntu_type}-${arch}" +fi + mkdir -p "$cache_dir" "$packer_cache_dir" export DEV_ALCHEMY_APP_DATA_DIR="$app_data_dir" export DEV_ALCHEMY_CACHE_DIR="$cache_dir" @@ -155,15 +169,16 @@ if [ "$arch" = "arm64" ]; then fi # remove packer output directory if it exists -output_dir="$cache_dir/ubuntu/qemu-out-ubuntu-${ubuntu_type}-${arch}" +output_dir="$build_output_dir" if [ -d "$output_dir" ]; then echo "Removing existing Packer output directory..." rm -rf "$output_dir" fi +mkdir -p "$(dirname "$output_dir")" packer init "build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl" if [ "$verbose" = "true" ]; then export PACKER_LOG=1 fi -packer build -var "cache_dir=$cache_dir" -var "iso_url=$iso_path" -var "ubuntu_type=$ubuntu_type" -var "headless=$headless" -var "vnc_port=$vnc_port" -var "arch=$arch" -var "cpus=$cpus" -var "memory=$memory" "build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl" +packer build -var "cache_dir=$cache_dir" -var "build_output_dir=$build_output_dir" -var "iso_url=$iso_path" -var "ubuntu_type=$ubuntu_type" -var "headless=$headless" -var "vnc_port=$vnc_port" -var "arch=$arch" -var "cpus=$cpus" -var "memory=$memory" "build/packer/linux/ubuntu/linux-ubuntu-on-macos.pkr.hcl" diff --git a/build/packer/windows/windows11-on-macos.pkr.hcl b/build/packer/windows/windows11-on-macos.pkr.hcl index 93fab6c2..f1b64470 100644 --- a/build/packer/windows/windows11-on-macos.pkr.hcl +++ b/build/packer/windows/windows11-on-macos.pkr.hcl @@ -55,6 +55,12 @@ variable "cache_dir" { } } +variable "build_output_dir" { + type = string + default = "" + description = "Optional short-lived Packer output directory to avoid long UNIX socket paths on macOS." +} + variable "is_ci" { type = bool default = env("CI") == "true" @@ -116,7 +122,7 @@ source "qemu" "win11" { headless = var.headless iso_url = local.win11_iso iso_checksum = "none" - output_directory = "${local.cache_directory}/windows11/qemu-out-windows11-${var.arch}" + output_directory = var.build_output_dir != "" ? var.build_output_dir : "${local.cache_directory}/windows11/qemu-out-windows11-${var.arch}" display = var.headless ? "none" : "cocoa" memory = var.memory cores = var.cpus diff --git a/build/packer/windows/windows11-on-macos.sh b/build/packer/windows/windows11-on-macos.sh index b1d445d3..5219f01b 100644 --- a/build/packer/windows/windows11-on-macos.sh +++ b/build/packer/windows/windows11-on-macos.sh @@ -9,6 +9,7 @@ vnc_port="5901" cpus="4" memory="4096" verbose="false" +build_output_dir="" script_dir=$( # shellcheck disable=SC2164 @@ -44,6 +45,15 @@ while [[ $# -gt 0 ]]; do exit 1 fi ;; + --build-output-dir) + if [[ -n "$2" ]]; then + build_output_dir="$2" + shift 2 + else + echo "Invalid value for --build-output-dir: $2." >&2 + exit 1 + fi + ;; --headless) headless="true" shift @@ -87,6 +97,10 @@ while [[ $# -gt 0 ]]; do esac done +if [[ -z "$build_output_dir" ]]; then + build_output_dir="/tmp/dev-alchemy/qemu-out-windows11-${arch}" +fi + echo "Using architecture: $arch" echo "Headless mode: $headless" @@ -166,16 +180,17 @@ elif [ "$arch" = "arm64" ]; then fi # remove packer output directory if it exists -output_dir="${cache_dir}/windows11/qemu-out-windows11-${arch}" +output_dir="${build_output_dir}" if [ -d "$output_dir" ]; then echo "Removing existing Packer output directory..." rm -rf "$output_dir" fi +mkdir -p "$(dirname "$output_dir")" if [ "$verbose" = "true" ]; then export PACKER_LOG=1 fi -packer build -var "cache_dir=${cache_dir}" -var "iso_url=${win11_iso_path}" -var "headless=$headless" -var "vnc_port=$vnc_port" -var "arch=$arch" -var "cpus=$cpus" -var "memory=$memory" "build/packer/windows/windows11-on-macos.pkr.hcl" +packer build -var "cache_dir=${cache_dir}" -var "build_output_dir=${build_output_dir}" -var "iso_url=${win11_iso_path}" -var "headless=$headless" -var "vnc_port=$vnc_port" -var "arch=$arch" -var "cpus=$cpus" -var "memory=$memory" "build/packer/windows/windows11-on-macos.pkr.hcl" packer_exit_code=$? exit $packer_exit_code diff --git a/pkg/build/external_process_handler.go b/pkg/build/external_process_handler.go index c20ef5c0..0a2821bc 100644 --- a/pkg/build/external_process_handler.go +++ b/pkg/build/external_process_handler.go @@ -146,6 +146,9 @@ func RunExternalProcessWithRetries(config RunProcessConfig) context.Context { case sig := <-sigs: log.Printf("Retry loop for process %s interrupted by signal: %v", config.ExecutablePath, sig) return cancelledContext() + case <-config.InterruptRetryChan: + log.Printf("Received interrupt signal while waiting to retry process %s", config.ExecutablePath) + return cancelledContext() case <-time.After(config.RetryInterval): } } @@ -157,6 +160,14 @@ func RunExternalProcessWithRetries(config RunProcessConfig) context.Context { return cancelledContext() default: } + if config.InterruptRetryChan != nil { + select { + case <-config.InterruptRetryChan: + log.Printf("Received interrupt signal before starting process %s attempt %d", config.ExecutablePath, attempt) + return cancelledContext() + default: + } + } ctx, err := RunExternalProcess(config) if err == nil { diff --git a/pkg/build/external_process_handler_test.go b/pkg/build/external_process_handler_test.go new file mode 100644 index 00000000..0b909ddf --- /dev/null +++ b/pkg/build/external_process_handler_test.go @@ -0,0 +1,42 @@ +package build + +import ( + "context" + "testing" + "time" +) + +func TestRunExternalProcessWithRetriesStopsOnInterrupt(t *testing.T) { + interrupt := make(chan bool, 1) + done := make(chan context.Context, 1) + start := time.Now() + + go func() { + done <- RunExternalProcessWithRetries(RunProcessConfig{ + ExecutablePath: "bash", + Args: []string{"-lc", "exit 1"}, + Timeout: 5 * time.Second, + Retries: 3, + RetryInterval: 5 * time.Second, + InterruptRetryChan: interrupt, + }) + }() + + time.Sleep(150 * time.Millisecond) + interrupt <- true + + select { + case ctx := <-done: + if ctx == nil { + t.Fatal("expected a cancelled context, got nil") + } + if ctx.Err() != context.Canceled { + t.Fatalf("expected context canceled, got %v", ctx.Err()) + } + if elapsed := time.Since(start); elapsed >= 2*time.Second { + t.Fatalf("expected retry loop to stop quickly after interrupt, took %s", elapsed) + } + case <-time.After(3 * time.Second): + t.Fatal("timed out waiting for retry loop to stop") + } +} diff --git a/pkg/build/helper.go b/pkg/build/helper.go index 431cda6a..b5a9ec49 100644 --- a/pkg/build/helper.go +++ b/pkg/build/helper.go @@ -93,3 +93,7 @@ func createHypervTempDir(dirs *Directories) error { // #nosec G301 -- this cache directory must remain traversable for non-root CI steps after sudo-created builds. return os.MkdirAll(tempPath, 0755) } + +func getDarwinQemuBuildOutputDir(config VirtualMachineConfig) string { + return filepath.Join("/tmp", "dev-alchemy", "qemu-out-"+GenerateVirtualMachineSlug(&config)) +} diff --git a/pkg/build/helper_test.go b/pkg/build/helper_test.go new file mode 100644 index 00000000..b7f07af7 --- /dev/null +++ b/pkg/build/helper_test.go @@ -0,0 +1,20 @@ +package build + +import ( + "path/filepath" + "testing" +) + +func TestGetDarwinQemuBuildOutputDir(t *testing.T) { + config := VirtualMachineConfig{ + OS: "ubuntu", + Arch: "amd64", + UbuntuType: "server", + } + + got := getDarwinQemuBuildOutputDir(config) + want := filepath.Join("/tmp", "dev-alchemy", "qemu-out-ubuntu-server-amd64") + if got != want { + t.Fatalf("expected %q, got %q", want, got) + } +} diff --git a/pkg/build/macos-silicon-build.go b/pkg/build/macos-silicon-build.go index cdb5020b..df3bede6 100644 --- a/pkg/build/macos-silicon-build.go +++ b/pkg/build/macos-silicon-build.go @@ -127,6 +127,18 @@ func RunFfmpegVideoGenerationProcess(vm_config VirtualMachineConfig, ctx context if len(inputPattern) > 4 && inputPattern[len(inputPattern)-4:] == ".jpg" { inputPattern = inputPattern[:len(inputPattern)-4] + "%05d.jpg" } + framePattern := recording_config.OutputFile + if len(framePattern) > 4 && framePattern[len(framePattern)-4:] == ".jpg" { + framePattern = framePattern[:len(framePattern)-4] + "*.jpg" + } + frames, err := filepath.Glob(framePattern) + if err != nil { + log.Fatalf("Failed to glob VNC snapshot frames: %v", err) + } + if len(frames) == 0 { + log.Printf("Skipping ffmpeg video generation because no VNC snapshot frames were captured.") + return ctx + } config := RunProcessConfig{ ExecutablePath: "ffmpeg", @@ -177,10 +189,7 @@ func RunFfmpegVideoGenerationProcess(vm_config VirtualMachineConfig, ctx context } // remove the snapshot images after video generation - removePattern := recording_config.OutputFile - if len(removePattern) > 4 && removePattern[len(removePattern)-4:] == ".jpg" { - removePattern = removePattern[:len(removePattern)-4] + "*.jpg" - } + removePattern := framePattern log.Printf("Removing vncsnapshot images with pattern: %s", removePattern) @@ -200,7 +209,16 @@ func RunFfmpegVideoGenerationProcess(vm_config VirtualMachineConfig, ctx context func RunQemuUbuntuBuildOnMacOS(config VirtualMachineConfig) error { scriptPath := filepath.Join(GetDirectoriesInstance().GetDirectories().ProjectDir, "build/packer/linux/ubuntu/linux-ubuntu-on-macos.sh") - args := []string{scriptPath, "--project-root", GetDirectoriesInstance().GetDirectories().ProjectDir, "--arch", config.Arch, "--ubuntu-type", config.UbuntuType, "--vnc-port", fmt.Sprintf("%d", config.VncPort), "--cpus", getVmCpuCountString(config), "--memory", fmt.Sprintf("%d", getVmMemoryMB(config))} + args := []string{ + scriptPath, + "--project-root", GetDirectoriesInstance().GetDirectories().ProjectDir, + "--build-output-dir", getDarwinQemuBuildOutputDir(config), + "--arch", config.Arch, + "--ubuntu-type", config.UbuntuType, + "--vnc-port", fmt.Sprintf("%d", config.VncPort), + "--cpus", getVmCpuCountString(config), + "--memory", fmt.Sprintf("%d", getVmMemoryMB(config)), + } if config.Headless { args = append(args, "--headless") } @@ -209,7 +227,15 @@ func RunQemuUbuntuBuildOnMacOS(config VirtualMachineConfig) error { func RunQemuWindowsBuildOnMacOS(config VirtualMachineConfig) error { scriptPath := filepath.Join(GetDirectoriesInstance().GetDirectories().ProjectDir, "build/packer/windows/windows11-on-macos.sh") - args := []string{scriptPath, "--project-root", GetDirectoriesInstance().GetDirectories().ProjectDir, "--arch", config.Arch, "--vnc-port", fmt.Sprintf("%d", config.VncPort), "--cpus", getVmCpuCountString(config), "--memory", fmt.Sprintf("%d", getVmMemoryMB(config))} + args := []string{ + scriptPath, + "--project-root", GetDirectoriesInstance().GetDirectories().ProjectDir, + "--build-output-dir", getDarwinQemuBuildOutputDir(config), + "--arch", config.Arch, + "--vnc-port", fmt.Sprintf("%d", config.VncPort), + "--cpus", getVmCpuCountString(config), + "--memory", fmt.Sprintf("%d", getVmMemoryMB(config)), + } if config.Headless { args = append(args, "--headless") } From 2bc4f7de660dd7112ce6bf85a5ed759c916a8c65 Mon Sep 17 00:00:00 2001 From: Carl-Christian Sautter Date: Sat, 28 Mar 2026 12:37:06 +0000 Subject: [PATCH 04/10] fix(build): restrict managed app directory permissions to 0700 --- pkg/build/directories.go | 3 ++- pkg/build/directories_test.go | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pkg/build/directories.go b/pkg/build/directories.go index ecc75d18..e024a4d0 100644 --- a/pkg/build/directories.go +++ b/pkg/build/directories.go @@ -14,6 +14,7 @@ const ( devAlchemyCacheEnvVar = "DEV_ALCHEMY_CACHE_DIR" devAlchemyVagrantEnvVar = "DEV_ALCHEMY_VAGRANT_DIR" devAlchemyPackerCacheEnvVar = "DEV_ALCHEMY_PACKER_CACHE_DIR" + managedDirPermission = 0o700 ) type Directories struct { @@ -182,7 +183,7 @@ func ensureDirectoriesExist(paths ...string) error { if currentPath == "" { continue } - if err := os.MkdirAll(currentPath, 0755); err != nil { + if err := os.MkdirAll(currentPath, managedDirPermission); err != nil { return err } } diff --git a/pkg/build/directories_test.go b/pkg/build/directories_test.go index 315efd0a..ac9cb2a2 100644 --- a/pkg/build/directories_test.go +++ b/pkg/build/directories_test.go @@ -2,6 +2,7 @@ package build import ( "errors" + "os" "path/filepath" "testing" ) @@ -115,3 +116,23 @@ func TestResolveDefaultAppDataDirForOS_ReturnsErrorWhenHomeUnavailable(t *testin t.Fatal("expected error when home directory lookup fails") } } + +func TestEnsureDirectoriesExist_CreatesPrivateDirectories(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "app", "cache") + + if err := ensureDirectoriesExist("", target); err != nil { + t.Fatalf("ensureDirectoriesExist returned error: %v", err) + } + + info, err := os.Stat(target) + if err != nil { + t.Fatalf("expected directory to exist: %v", err) + } + if !info.IsDir() { + t.Fatalf("expected %q to be a directory", target) + } + if got := info.Mode().Perm(); got != managedDirPermission { + t.Fatalf("expected permissions %o, got %o", managedDirPermission, got) + } +} From 4db3625756827014e0a124f8b4594b6c8c1d1f6c Mon Sep 17 00:00:00 2001 From: Carl-Christian Sautter Date: Sat, 28 Mar 2026 12:43:13 +0000 Subject: [PATCH 05/10] ci: run gitleaks and cmd unit tests on main pushes --- .github/workflows/gitleaks.yaml | 2 ++ .github/workflows/test-cmd-unit-tests.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/gitleaks.yaml b/.github/workflows/gitleaks.yaml index 68663f32..0df7b63c 100644 --- a/.github/workflows/gitleaks.yaml +++ b/.github/workflows/gitleaks.yaml @@ -2,6 +2,8 @@ name: gitleaks on: pull_request: push: + branches: + - main workflow_dispatch: schedule: - cron: "0 4 * * *" # run once a day at 4 AM diff --git a/.github/workflows/test-cmd-unit-tests.yml b/.github/workflows/test-cmd-unit-tests.yml index d3ba3f2f..d292d1ec 100644 --- a/.github/workflows/test-cmd-unit-tests.yml +++ b/.github/workflows/test-cmd-unit-tests.yml @@ -2,6 +2,8 @@ name: Test Go Unit Tests on: push: + branches: + - main paths: - "cmd/cmd/**" - ".github/workflows/test-cmd-unit-tests.yml" From ec0dfc480fe12bdcdc7d8e3c72a4585a4d6d832f Mon Sep 17 00:00:00 2001 From: Carl-Christian Sautter Date: Sat, 28 Mar 2026 12:50:28 +0000 Subject: [PATCH 06/10] test(build): skip POSIX permission assertion on Windows --- pkg/build/directories_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/build/directories_test.go b/pkg/build/directories_test.go index ac9cb2a2..14d5bd3d 100644 --- a/pkg/build/directories_test.go +++ b/pkg/build/directories_test.go @@ -4,6 +4,7 @@ import ( "errors" "os" "path/filepath" + "runtime" "testing" ) @@ -132,6 +133,9 @@ func TestEnsureDirectoriesExist_CreatesPrivateDirectories(t *testing.T) { if !info.IsDir() { t.Fatalf("expected %q to be a directory", target) } + if runtime.GOOS == "windows" { + return + } if got := info.Mode().Perm(); got != managedDirPermission { t.Fatalf("expected permissions %o, got %o", managedDirPermission, got) } From 23dc24ffff8f76e7da0ccde7d6dad99d063181c9 Mon Sep 17 00:00:00 2001 From: Carl-Christian Sautter Date: Sat, 28 Mar 2026 12:54:16 +0000 Subject: [PATCH 07/10] ci: run deploy and provision unit tests on main pushes --- .github/workflows/test-deploy-unit-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-deploy-unit-tests.yml b/.github/workflows/test-deploy-unit-tests.yml index 408f3846..272aa601 100644 --- a/.github/workflows/test-deploy-unit-tests.yml +++ b/.github/workflows/test-deploy-unit-tests.yml @@ -2,6 +2,8 @@ name: Test Deploy and Provision Unit Tests on: push: + branches: + - main paths: - "pkg/deploy/**" - "pkg/provision/**" From 0e082708048154c32f3b4e0e91de756805f04aba Mon Sep 17 00:00:00 2001 From: Carl-Christian Sautter Date: Sat, 28 Mar 2026 16:28:28 +0000 Subject: [PATCH 08/10] fix(ci): align macOS workflow cache paths with managed app data --- .github/workflows/test-build-macos.yml | 40 +++++++++++++++----------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-build-macos.yml b/.github/workflows/test-build-macos.yml index 4b0cf00d..81e2025d 100644 --- a/.github/workflows/test-build-macos.yml +++ b/.github/workflows/test-build-macos.yml @@ -29,6 +29,12 @@ jobs: if: github.event_name == 'pull_request' || inputs.enable_utm_qemu_macos name: build ${{ matrix.go_test_name }} VM_OS ${{ matrix.vm_os }} runner_os ${{ matrix.runs_on }} runs-on: ${{ matrix.runs_on }} + env: + DEV_ALCHEMY_APP_DATA_DIR: ${{ github.workspace }}/.dev-alchemy + DEV_ALCHEMY_CACHE_DIR: ${{ github.workspace }}/.dev-alchemy/cache + DEV_ALCHEMY_VAGRANT_DIR: ${{ github.workspace }}/.dev-alchemy/.vagrant + DEV_ALCHEMY_PACKER_CACHE_DIR: ${{ github.workspace }}/.dev-alchemy/packer_cache + PACKER_CACHE_DIR: ${{ github.workspace }}/.dev-alchemy/packer_cache timeout-minutes: 260 strategy: fail-fast: false @@ -39,43 +45,43 @@ jobs: deploy_test_name: TestDeployUtmWindows11Arm64OnMacos runs_on: macos-26-tart cache_files: >- - [{"local-path":"./cache/windows11/iso/win11_25h2_english_arm64.iso","container":"build-cache"}, - {"local-path":"./cache/utm/utm-guest-tools-latest.iso"}, - {"local-path":"./cache/windows/virtio-win.iso"}, - {"local-path":"./cache/qemu-efi-aarch64_all.deb"}] + [{"local-path":"./.dev-alchemy/cache/windows11/iso/win11_25h2_english_arm64.iso","container":"build-cache"}, + {"local-path":"./.dev-alchemy/cache/utm/utm-guest-tools-latest.iso"}, + {"local-path":"./.dev-alchemy/cache/windows/virtio-win.iso"}, + {"local-path":"./.dev-alchemy/cache/qemu-efi-aarch64_all.deb"}] - vm_os: windows11 go_test_name: TestBuildQemuWindows11Amd64OnMacos deploy_test_name: TestDeployUtmWindows11Amd64OnMacos runs_on: macos-26-tart cache_files: >- - [{"local-path":"./cache/windows11/iso/win11_25h2_english_amd64.iso","container":"build-cache"}, - {"local-path":"./cache/utm/utm-guest-tools-latest.iso"}] + [{"local-path":"./.dev-alchemy/cache/windows11/iso/win11_25h2_english_amd64.iso","container":"build-cache"}, + {"local-path":"./.dev-alchemy/cache/utm/utm-guest-tools-latest.iso"}] - vm_os: ubuntu go_test_name: TestBuildQemuUbuntuServerArm64OnMacos deploy_test_name: TestDeployUtmUbuntuServerArm64OnMacos runs_on: macos-26-tart cache_files: >- - [{"local-path":"./cache/linux/ubuntu-24.04.3-live-server-arm64.iso"}, - {"local-path":"./cache/qemu-efi-aarch64_all.deb"}] + [{"local-path":"./.dev-alchemy/cache/linux/ubuntu-24.04.3-live-server-arm64.iso"}, + {"local-path":"./.dev-alchemy/cache/qemu-efi-aarch64_all.deb"}] - vm_os: ubuntu go_test_name: TestBuildQemuUbuntuServerAmd64OnMacos deploy_test_name: TestDeployUtmUbuntuServerAmd64OnMacos runs_on: macos-26-tart cache_files: >- - [{"local-path":"./cache/linux/ubuntu-24.04.3-live-server-amd64.iso"}] + [{"local-path":"./.dev-alchemy/cache/linux/ubuntu-24.04.3-live-server-amd64.iso"}] - vm_os: ubuntu go_test_name: TestBuildQemuUbuntuDesktopArm64OnMacos deploy_test_name: TestDeployUtmUbuntuDesktopArm64OnMacos runs_on: macos-26-tart cache_files: >- - [{"local-path":"./cache/linux/ubuntu-24.04.3-live-server-arm64.iso"}, - {"local-path":"./cache/qemu-efi-aarch64_all.deb"}] + [{"local-path":"./.dev-alchemy/cache/linux/ubuntu-24.04.3-live-server-arm64.iso"}, + {"local-path":"./.dev-alchemy/cache/qemu-efi-aarch64_all.deb"}] - vm_os: ubuntu go_test_name: TestBuildQemuUbuntuDesktopAmd64OnMacos deploy_test_name: TestDeployUtmUbuntuDesktopAmd64OnMacos runs_on: macos-26-tart cache_files: >- - [{"local-path":"./cache/linux/ubuntu-24.04.3-live-server-amd64.iso"}] + [{"local-path":"./.dev-alchemy/cache/linux/ubuntu-24.04.3-live-server-amd64.iso"}] steps: - name: Checkout code uses: actions/checkout@v6 @@ -116,11 +122,11 @@ jobs: go mod vendor sudo -E make test-build-specific TEST_NAME=${{ matrix.go_test_name }} - - name: Normalize cache ownership after sudo build + - name: Normalize managed app-data ownership after sudo build if: always() && contains(matrix.runs_on, 'macos') run: | - if [ -d ./cache ]; then - sudo chown -R "$(id -u):$(id -g)" ./cache + if [ -d ./.dev-alchemy ]; then + sudo chown -R "$(id -u):$(id -g)" ./.dev-alchemy fi - name: Run deploy smoke test TEST_NAME=${{ matrix.deploy_test_name }} @@ -138,7 +144,9 @@ jobs: if: contains(matrix.runs_on, 'macos') run: | mkdir -p artifact - find ./cache -name '*.vnc.mp4' -exec cp {} artifact/ \; + if [ -d ./.dev-alchemy/cache ]; then + find ./.dev-alchemy/cache -name '*.vnc.mp4' -exec cp {} artifact/ \; + fi - name: Upload packer-qemu-${{ matrix.vm_os }}-${{ matrix.go_test_name }}.vnc.mp4 artifact (always) if: contains(matrix.runs_on, 'macos') From c0e419a43d7d0de214d07fc36733ac415a89d8a0 Mon Sep 17 00:00:00 2001 From: Carl-Christian Sautter Date: Sat, 28 Mar 2026 16:36:00 +0000 Subject: [PATCH 09/10] fix(ci): align Windows workflow cache paths with managed app data --- .github/workflows/test-build-windows.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-build-windows.yml b/.github/workflows/test-build-windows.yml index f17b3aca..e2a35be5 100644 --- a/.github/workflows/test-build-windows.yml +++ b/.github/workflows/test-build-windows.yml @@ -102,6 +102,11 @@ jobs: needs: [provision-hyperv-runner] timeout-minutes: 60 env: + DEV_ALCHEMY_APP_DATA_DIR: ${{ github.workspace }}\.dev-alchemy + DEV_ALCHEMY_CACHE_DIR: ${{ github.workspace }}\.dev-alchemy\cache + DEV_ALCHEMY_VAGRANT_DIR: ${{ github.workspace }}\.dev-alchemy\.vagrant + DEV_ALCHEMY_PACKER_CACHE_DIR: ${{ github.workspace }}\.dev-alchemy\packer_cache + PACKER_CACHE_DIR: ${{ github.workspace }}\.dev-alchemy\packer_cache PACKER_TEMP_PATH: "D:\\hyperv-temp" # Use Azure VM temp disk for packer temp files to speed up build runs-on: - self-hosted @@ -124,7 +129,7 @@ jobs: - name: Download build cache files uses: ./.github/actions/download-build-cache with: - files: '[{"local-path":"./cache/windows11/iso/win11_25h2_english_amd64.iso","container":"build-cache"}]' + files: '[{"local-path":"./.dev-alchemy/cache/windows11/iso/win11_25h2_english_amd64.iso","container":"build-cache"}]' subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: Install dependencies windows @@ -172,7 +177,7 @@ jobs: - name: Upload build cache files uses: ./.github/actions/upload-build-cache with: - files: '[{"local-path":"./cache/windows11/iso/win11_25h2_english_amd64.iso","container":"build-cache"}]' + files: '[{"local-path":"./.dev-alchemy/cache/windows11/iso/win11_25h2_english_amd64.iso","container":"build-cache"}]' subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: Fail if build or deploy smoke test failed @@ -298,6 +303,11 @@ jobs: needs: [provision-virtualbox-runner] timeout-minutes: 80 env: + DEV_ALCHEMY_APP_DATA_DIR: ${{ github.workspace }}\.dev-alchemy + DEV_ALCHEMY_CACHE_DIR: ${{ github.workspace }}\.dev-alchemy\cache + DEV_ALCHEMY_VAGRANT_DIR: ${{ github.workspace }}\.dev-alchemy\.vagrant + DEV_ALCHEMY_PACKER_CACHE_DIR: ${{ github.workspace }}\.dev-alchemy\packer_cache + PACKER_CACHE_DIR: ${{ github.workspace }}\.dev-alchemy\packer_cache PACKER_TEMP_PATH: "D:\\vbox-temp" # Use Azure VM temp disk for packer temp files to speed up build runs-on: - self-hosted @@ -320,7 +330,7 @@ jobs: - name: Download build cache files uses: ./.github/actions/download-build-cache with: - files: '[{"local-path":"./cache/windows11/iso/win11_25h2_english_amd64.iso","container":"build-cache"}]' + files: '[{"local-path":"./.dev-alchemy/cache/windows11/iso/win11_25h2_english_amd64.iso","container":"build-cache"}]' subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: Install dependencies windows @@ -358,7 +368,7 @@ jobs: - name: Upload build cache files uses: ./.github/actions/upload-build-cache with: - files: '[{"local-path":"./cache/windows11/iso/win11_25h2_english_amd64.iso","container":"build-cache"}]' + files: '[{"local-path":"./.dev-alchemy/cache/windows11/iso/win11_25h2_english_amd64.iso","container":"build-cache"}]' subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: Fail if Packer build failed From 99b0cf390348d4d4c09ee48165273a0e0c1ba099 Mon Sep 17 00:00:00 2001 From: Carl-Christian Sautter Date: Sat, 28 Mar 2026 16:47:44 +0000 Subject: [PATCH 10/10] fix(ci): harden Playwright Windows ISO fetch and upload diagnostics --- .github/workflows/test-build-macos.yml | 9 + scripts/macos/README.md | 4 + scripts/macos/playwright_win11_iso.py | 376 +++++++++++++++++++------ 3 files changed, 306 insertions(+), 83 deletions(-) diff --git a/.github/workflows/test-build-macos.yml b/.github/workflows/test-build-macos.yml index 81e2025d..8758afb4 100644 --- a/.github/workflows/test-build-macos.yml +++ b/.github/workflows/test-build-macos.yml @@ -148,6 +148,15 @@ jobs: find ./.dev-alchemy/cache -name '*.vnc.mp4' -exec cp {} artifact/ \; fi + - name: Upload Playwright diagnostics artifact (always) + if: always() && contains(matrix.runs_on, 'macos') + uses: actions/upload-artifact@v7 + with: + name: playwright-diagnostics-${{ matrix.vm_os }}-${{ matrix.go_test_name }} + path: ./.dev-alchemy/cache/windows/playwright-diagnostics/** + if-no-files-found: ignore + retention-days: 7 + - name: Upload packer-qemu-${{ matrix.vm_os }}-${{ matrix.go_test_name }}.vnc.mp4 artifact (always) if: contains(matrix.runs_on, 'macos') uses: actions/upload-artifact@v7 diff --git a/scripts/macos/README.md b/scripts/macos/README.md index 51fdee10..8d2b12a4 100644 --- a/scripts/macos/README.md +++ b/scripts/macos/README.md @@ -42,9 +42,13 @@ Additionally, the script saves the download link to the managed cache directory: - macOS default: `~/Library/Application Support/dev-alchemy/cache/windows/win11_amd64_iso_url.txt` - macOS default: `~/Library/Application Support/dev-alchemy/cache/windows/win11_arm64_iso_url.txt` +- Session cookies: `~/Library/Application Support/dev-alchemy/cache/windows/playwright/cookies.json` +- Failure diagnostics: `~/Library/Application Support/dev-alchemy/cache/windows/playwright-diagnostics/` Set `DEV_ALCHEMY_APP_DATA_DIR` to move the managed cache elsewhere. +If Microsoft serves a slow, blocked, or challenge page, the script now retries the initial navigation and writes a screenshot, HTML snapshot, and JSON metadata bundle under `playwright-diagnostics/` before failing. + ### Download the ISO You can use `curl` or `wget` to download the ISO using the link saved in the file: ```bash diff --git a/scripts/macos/playwright_win11_iso.py b/scripts/macos/playwright_win11_iso.py index 7c97ff3d..6c0e464b 100644 --- a/scripts/macos/playwright_win11_iso.py +++ b/scripts/macos/playwright_win11_iso.py @@ -2,16 +2,19 @@ Fetches the latest Windows 11 ISO download link from Microsoft's official download page using Playwright. """ -import asyncio -from playwright.async_api import async_playwright - -from playwright_stealth import Stealth -import random -import time import argparse -import os +import asyncio import json +import os +import random import sys +import time +from datetime import datetime, timezone + +from playwright.async_api import Error as PlaywrightError +from playwright.async_api import TimeoutError as PlaywrightTimeoutError +from playwright.async_api import async_playwright +from playwright_stealth import Stealth MICROSOFT_WIN11_ISO_URL = "https://www.microsoft.com/en-US/software-download/windows11" MICROSOFT_WIN11_ARM_ISO_URL = ( @@ -35,6 +38,21 @@ "Mozilla/5.0 (Macintosh; Intel Mac OS X 15_7_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15", ] +DEFAULT_NAVIGATION_TIMEOUT_MS = 90000 +DEFAULT_PAGE_READY_TIMEOUT_MS = 60000 +DEFAULT_NAVIGATION_RETRIES = 3 +EXPECTED_PAGE_SELECTOR = "#product-edition" +CHALLENGE_MARKERS = ( + "verify you are human", + "unusual traffic", + "access denied", + "temporarily unavailable", + "captcha", + "blocked", + "bot", + "challenge", +) + def resolve_app_data_dir() -> str: override = os.environ.get("DEV_ALCHEMY_APP_DATA_DIR") @@ -69,6 +87,100 @@ def resolve_cache_dir() -> str: return os.path.join(resolve_app_data_dir(), "cache") +def resolve_windows_cache_dir(*parts: str) -> str: + return os.path.join(resolve_cache_dir(), "windows", *parts) + + +def resolve_cookie_path() -> str: + return resolve_windows_cache_dir("playwright", "cookies.json") + + +def resolve_diagnostics_root() -> str: + return resolve_windows_cache_dir("playwright-diagnostics") + + +def utc_timestamp() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def ensure_parent_dir(path: str) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + + +def detect_challenge_markers(text: str) -> list[str]: + lowered = text.lower() + return [marker for marker in CHALLENGE_MARKERS if marker in lowered] + + +async def gather_page_state(page) -> dict: + metadata = { + "current_url": "", + "title": "", + "ready_state": "", + "challenge_markers": [], + } + body_text = "" + + try: + metadata["current_url"] = page.url + except Exception: + pass + + try: + metadata["title"] = await page.title() + except Exception: + pass + + try: + metadata["ready_state"] = await page.evaluate("() => document.readyState") + except Exception: + pass + + try: + body_text = await page.locator("body").inner_text(timeout=5000) + except Exception: + pass + + combined_text = "\n".join( + value + for value in (metadata["title"], metadata["current_url"], body_text) + if value + ) + metadata["challenge_markers"] = detect_challenge_markers(combined_text) + return metadata + + +async def write_failure_artifacts(page, diagnostics_dir: str, label: str) -> dict: + os.makedirs(diagnostics_dir, exist_ok=True) + + metadata = await gather_page_state(page) + metadata_path = os.path.join(diagnostics_dir, f"{label}.json") + html_path = os.path.join(diagnostics_dir, f"{label}.html") + screenshot_path = os.path.join(diagnostics_dir, f"{label}.png") + + try: + ensure_parent_dir(metadata_path) + with open(metadata_path, "w", encoding="utf-8") as f: + json.dump(metadata, f, indent=2, sort_keys=True) + except Exception as exc: + print(f"[WARN] Failed to write metadata artifact {metadata_path}: {exc}") + + try: + html = await page.content() + ensure_parent_dir(html_path) + with open(html_path, "w", encoding="utf-8") as f: + f.write(html) + except Exception as exc: + print(f"[WARN] Failed to write HTML artifact {html_path}: {exc}") + + try: + await page.screenshot(path=screenshot_path, full_page=True) + except Exception as exc: + print(f"[WARN] Failed to write screenshot artifact {screenshot_path}: {exc}") + + return metadata + + async def random_mouse_movements(page, min_seconds=5, max_seconds=15): duration = random.uniform(min_seconds, max_seconds) start_time = time.time() @@ -104,7 +216,8 @@ async def dismiss_modal_if_present(page): async def click_button_with_retry(page, selector, selector_condition, retries=10): - for _ in range(retries): + last_error = None + for attempt in range(1, retries + 1): try: await page.wait_for_selector(selector, timeout=60000) button = page.locator(selector) @@ -113,13 +226,16 @@ async def click_button_with_retry(page, selector, selector_condition, retries=10 else: print(f"{selector} button is not visible.") - except Exception: - pass + except Exception as exc: + last_error = exc + print( + f"[WARN] Attempt {attempt}/{retries} failed while clicking {selector}: {exc}" + ) try: - await page.wait_for_selector(selector_condition) - except Exception: - pass + await page.wait_for_selector(selector_condition, timeout=30000) + except Exception as exc: + last_error = exc expected = page.locator(selector_condition) if expected and await expected.is_visible(timeout=30000): @@ -127,7 +243,9 @@ async def click_button_with_retry(page, selector, selector_condition, retries=10 else: await dismiss_modal_if_present(page) - return False + raise RuntimeError( + f"Failed to advance from {selector} to {selector_condition} after {retries} attempts." + ) from last_error async def select_option_by_text(page, selector, text_match): @@ -141,6 +259,67 @@ async def select_option_by_text(page, selector, text_match): return None +async def navigate_to_download_page( + page, + url: str, + diagnostics_dir: str, + navigation_timeout_ms: int, + page_ready_timeout_ms: int, + retries: int, +): + last_error = None + last_metadata = None + + for attempt in range(1, retries + 1): + print( + f"[INFO] Navigating to {url} (attempt {attempt}/{retries}, timeout={navigation_timeout_ms}ms)" + ) + try: + response = await page.goto( + url, + timeout=navigation_timeout_ms, + wait_until="domcontentloaded", + ) + if response is not None: + print( + f"[INFO] Navigation response: status={response.status} url={response.url}" + ) + + await page.wait_for_selector( + EXPECTED_PAGE_SELECTOR, timeout=page_ready_timeout_ms + ) + return + except (PlaywrightTimeoutError, PlaywrightError) as exc: + last_error = exc + print(f"[WARN] Navigation attempt {attempt}/{retries} failed: {exc}") + last_metadata = await write_failure_artifacts( + page, diagnostics_dir, f"initial-navigation-attempt-{attempt}" + ) + if last_metadata.get("challenge_markers"): + print( + "[WARN] Page looks like a challenge/blocked response. " + f"Detected markers: {', '.join(last_metadata['challenge_markers'])}" + ) + + if attempt < retries: + sleep_seconds = min(20.0, (2**attempt) + random.uniform(0.5, 1.5)) + print(f"[INFO] Retrying navigation after {sleep_seconds:.1f}s") + await asyncio.sleep(sleep_seconds) + + challenge_hint = "" + if last_metadata and last_metadata.get("challenge_markers"): + challenge_hint = ( + " Possible anti-bot or blocked-page markers were detected: " + + ", ".join(last_metadata["challenge_markers"]) + + "." + ) + + raise RuntimeError( + "Could not load the Microsoft Windows 11 download page after " + f"{retries} attempts. Diagnostics were written to {diagnostics_dir}.{challenge_hint}" + ) from last_error + + async def fetch_win11_iso_link( arm: bool = False, headless: bool = False, @@ -148,7 +327,10 @@ async def fetch_win11_iso_link( save_path: str = "", ): async with Stealth().use_async(async_playwright()) as p: - browser = await p.chromium.launch(headless=headless) + browser = await p.chromium.launch( + headless=headless, + args=["--disable-blink-features=AutomationControlled"], + ) # Randomly select viewport size width = random.randint(800, 1920) @@ -159,13 +341,16 @@ async def fetch_win11_iso_link( context = await browser.new_context( locale="en-US", - timezone_id="EST", + timezone_id="America/New_York", user_agent=user_agent, viewport={"width": width, "height": height}, ) + context.set_default_timeout(DEFAULT_PAGE_READY_TIMEOUT_MS) + context.set_default_navigation_timeout(DEFAULT_NAVIGATION_TIMEOUT_MS) # Load cookies from a previous session + cookie_path = resolve_cookie_path() try: - with open("cookies.json", "r") as f: + with open(cookie_path, "r", encoding="utf-8") as f: saved_cookies = json.load(f) await context.add_cookies(saved_cookies) except (FileNotFoundError, json.JSONDecodeError): @@ -182,75 +367,100 @@ async def fetch_win11_iso_link( page = await context.new_page() url = MICROSOFT_WIN11_ARM_ISO_URL if arm else MICROSOFT_WIN11_ISO_URL - await page.goto(url) - - await random_mouse_movements(page) + diagnostics_dir = os.path.join( + resolve_diagnostics_root(), + f"{'arm64' if arm else 'amd64'}-{utc_timestamp()}", + ) - # Accept cookies if prompted try: - await page.click('button:has-text("Accept")', timeout=3000) + await navigate_to_download_page( + page=page, + url=url, + diagnostics_dir=diagnostics_dir, + navigation_timeout_ms=DEFAULT_NAVIGATION_TIMEOUT_MS, + page_ready_timeout_ms=DEFAULT_PAGE_READY_TIMEOUT_MS, + retries=DEFAULT_NAVIGATION_RETRIES, + ) + + await random_mouse_movements(page) + + # Accept cookies if prompted + try: + await page.click('button:has-text("Accept")', timeout=3000) + except Exception: + pass + + await random_mouse_movements(page) + + # Select 'Windows 11 (multi-edition ISO)' from the dropdown + selected_product = await select_option_by_text( + page, "#product-edition", "Windows 11" + ) + if selected_product is None: + raise RuntimeError( + "Could not find a Windows 11 product option on the Microsoft download page." + ) + + expect_creation_of_selector = "#product-languages" + await click_button_with_retry( + page, "#submit-product-edition", expect_creation_of_selector + ) + + await random_mouse_movements(page) + + # Select English (United States) + lang_value = await select_option_by_text( + page, "#product-languages", "English (United States)" + ) + if lang_value is None: + raise RuntimeError( + "Could not find 'English (United States)' language option." + ) + + await random_mouse_movements(page) + + # expected selector + download_selector = "#download-links > div > div > a:first-child" + await click_button_with_retry(page, "#submit-sku", download_selector) + + await random_mouse_movements(page) + + await page.wait_for_selector(download_selector, timeout=60000) + download_button = page.locator(download_selector) + link = await download_button.get_attribute("href") + if not link: + raise RuntimeError("Could not retrieve the download link.") + + print(f"Windows 11 ISO download link: {link}") + + # Write link to file for use in packer build + output_dir = resolve_windows_cache_dir() + os.makedirs(output_dir, exist_ok=True) + output_path = os.path.join( + output_dir, + "win11_arm64_iso_url.txt" if arm else "win11_amd64_iso_url.txt", + ) + with open(output_path, "w", encoding="utf-8") as f: + f.write(link) + + if download: + print("Starting ISO download...") + async with page.expect_download() as download_info: + await page.goto(link, wait_until="commit") + download_obj = await download_info.value + await download_obj.save_as(save_path) + print(f"ISO saved to {save_path}") + + # Save current session cookies for future use + current_cookies = await context.cookies() + ensure_parent_dir(cookie_path) + with open(cookie_path, "w", encoding="utf-8") as f: + json.dump(current_cookies, f) except Exception: - pass - - await random_mouse_movements(page) - - # Select 'Windows 11 (multi-edition ISO)' from the dropdown - await select_option_by_text(page, "#product-edition", "Windows 11") - - expect_creation_of_selector = "#product-languages" - await click_button_with_retry( - page, "#submit-product-edition", expect_creation_of_selector - ) - - await random_mouse_movements(page) - - # Select English (United States) - lang_value = await select_option_by_text( - page, "#product-languages", "English (United States)" - ) - if lang_value is None: - raise Exception("Could not find 'English (United States)' language option.") - - await random_mouse_movements(page) - - # expected selector - download_selector = "#download-links > div > div > a:first-child" - await click_button_with_retry(page, "#submit-sku", download_selector) - - await random_mouse_movements(page) - - await page.wait_for_selector(download_selector, timeout=60000) - download_button = page.locator(download_selector) - link = await download_button.get_attribute("href") - if not link: - raise Exception("Could not retrieve the download link.") - - print(f"Windows 11 ISO download link: {link}") - - # Write link to file for use in packer build - script_dir = os.path.dirname(os.path.abspath(__file__)) - output_dir = os.path.join(resolve_cache_dir(), "windows") - os.makedirs(output_dir, exist_ok=True) - output_path = os.path.join( - output_dir, "win11_arm64_iso_url.txt" if arm else "win11_amd64_iso_url.txt" - ) - with open(output_path, "w") as f: - f.write(link) - - if download: - print("Starting ISO download...") - async with page.expect_download() as download_info: - await page.goto(link) - download_obj = await download_info.value - await download_obj.save_as(save_path) - print(f"ISO saved to {save_path}") - - # Save current session cookies for future use - current_cookies = await context.cookies() - with open("cookies.json", "w") as f: - json.dump(current_cookies, f) - - await browser.close() + await write_failure_artifacts(page, diagnostics_dir, "fatal-error") + raise + finally: + await browser.close() if __name__ == "__main__":