diff --git a/build/ci/README.md b/build/ci/README.md index ea878c167fd..d23d974764a 100644 --- a/build/ci/README.md +++ b/build/ci/README.md @@ -7,6 +7,59 @@ Workflow descriptions are located in `.github/workflows`, platform specific PCP * QA: Runs the entire testsuite on pull requests and daily at 17:00 UTC, and publishes the results at https://performancecopilot.github.io/qa-reports/ * Release: Triggered when a new tag is pushed, creates a new release and pushes it to https://packagecloud.io/performancecopilot/pcp +## Quick Start - Running CI Locally + +Test your changes against a single platform: +```bash +# Full CI pipeline (setup → build → install → init_qa → qa_sanity) +build/ci/ci-run.py ubuntu2404-container reproduce --until qa_sanity + +# Just setup and build (quick feedback on compilation issues) +build/ci/ci-run.py ubuntu2404-container reproduce --until build + +# Run individual tasks +build/ci/ci-run.py ubuntu2404-container task build +build/ci/ci-run.py ubuntu2404-container task qa_sanity +``` + +## Supported Platforms for Local Testing + +See `.github/workflows/ci.yml` for the authoritative list of platforms being tested in CI. Currently tested platforms include: + +- ubuntu1804-i386-container (Ubuntu 18.04, 32-bit) +- ubuntu2004-container (Ubuntu 20.04) +- ubuntu2204-container (Ubuntu 22.04) +- ubuntu2404-container (Ubuntu 24.04 LTS, recommended for new development) +- fedora42-container (Fedora 42) +- fedora43-container (Fedora 43) +- centos-stream8-container (CentOS Stream 8) +- centos-stream9-container (CentOS Stream 9) +- centos-stream10-container (CentOS Stream 10) + +Additional platforms available in the codebase but not in the default CI matrix (see comments in `.github/workflows/ci.yml`): +- debian12-container +- debian13-container +- fedora-rawhide-container + +## Architecture Support + +By default, the CI scripts use your native host architecture: +- **macOS ARM64 (Apple Silicon)**: Runs native ARM64 containers (fast, no emulation) +- **macOS Intel**: Runs amd64 containers natively +- **Linux**: Always uses amd64 containers + +To force amd64 emulation on macOS for exact CI parity (requires Rosetta): +```bash +build/ci/ci-run.py --emulate ubuntu2404-container reproduce +``` + +## Troubleshooting (macOS) + +If you see "exec: Exec format error" with amd64 emulation, Rosetta may have become unstable. Restart the podman machine: +```bash +podman machine stop && podman machine start +``` + ## Reproducing test failures ``` build/ci/ci-run.py ubuntu2004-container|fedora38-container|centos9-container|... reproduce diff --git a/build/ci/ci-run.py b/build/ci/ci-run.py index 17a8da08934..2993edaae1f 100755 --- a/build/ci/ci-run.py +++ b/build/ci/ci-run.py @@ -5,9 +5,158 @@ import sys import os import tempfile +import platform as platform_module from datetime import datetime +def get_host_os(): + return platform_module.system() + + +def get_host_architecture(): + return platform_module.machine() + + +def is_macos(): + return get_host_os() == "Darwin" + + +def is_arm64(): + return get_host_architecture() in ("arm64", "aarch64") + + +def _parse_platform_string(value, separator): + """Parse a string into a list of platform names, handling both delimiters and empty values.""" + if not value: + return None + # Use separator as primary delimiter, then split on whitespace + platforms = [p.strip() for p in value.replace(separator, ' ').split() if p.strip()] + return platforms if platforms else None + + +def resolve_platforms_to_run(cli_platforms=None, env_platforms=None, config_file=None): + """ + Resolve platforms to run with priority hierarchy. + + Priority: + 1. CLI arguments (comma or space-separated) + 2. Environment variable PCP_CI_QUICK_PLATFORMS + 3. Config file .pcp-ci-quick (line-separated, comments ignored) + 4. None - requires user to specify platforms + + Args: + cli_platforms: String of platforms from command line + env_platforms: String of platforms from environment variable + config_file: Path to config file (default: .pcp-ci-quick) + + Returns: + List of platform names, or None if no platforms found + """ + if config_file is None: + config_file = ".pcp-ci-quick" + + # Priority 1: CLI arguments + platforms = _parse_platform_string(cli_platforms, ',') + if platforms: + return platforms + + # Priority 2: Environment variable + platforms = _parse_platform_string(env_platforms, ',') + if platforms: + return platforms + + # Priority 3: Config file + if os.path.isfile(config_file): + try: + with open(config_file, encoding="utf-8") as f: + platforms = [line.strip() for line in f if line.strip() and not line.strip().startswith('#')] + if platforms: + return platforms + except (IOError, OSError) as e: + print(f"Warning: Could not read config file {config_file}: {e}", file=sys.stderr) + + return None + + +def print_quick_mode_help(): + """Print helpful error message for quick mode.""" + print("\nError: No platforms specified for --quick mode.", file=sys.stderr) + print("\nPlease specify platforms using one of these methods:\n", file=sys.stderr) + print(" 1. Command line (comma or space-separated):", file=sys.stderr) + print(" python3 build/ci/ci-run.py --quick ubuntu2404-container,fedora43-container reproduce\n", file=sys.stderr) + print(" 2. Environment variable:", file=sys.stderr) + print(" export PCP_CI_QUICK_PLATFORMS='ubuntu2404-container fedora43-container'", file=sys.stderr) + print(" python3 build/ci/ci-run.py --quick reproduce\n", file=sys.stderr) + print(" 3. Config file (.pcp-ci-quick in repo root):", file=sys.stderr) + print(" ubuntu2404-container", file=sys.stderr) + print(" fedora43-container", file=sys.stderr) + print(" centos-stream10-container", file=sys.stderr) + print(" python3 build/ci/ci-run.py --quick reproduce\n", file=sys.stderr) + + +def _execute_command(runner, args, platform_name=None): + """ + Execute the appropriate command on the runner. + + Args: + runner: The runner instance (ContainerRunner, etc.) + args: Parsed command-line arguments + platform_name: Optional platform name for logging (used in quick mode) + """ + try: + if args.main_command == "setup": + runner.setup(args.pcp_path) + runner.task("setup") + elif args.main_command == "destroy": + runner.destroy() + elif args.main_command == "task": + runner.task(args.task_name) + elif args.main_command == "artifacts": + runner.task(f"copy_{args.artifact}_artifacts") + runner.get_artifacts(args.artifact, args.path) + elif args.main_command == "exec": + runner.exec(" ".join(args.command), check=False) + elif args.main_command == "shell": + runner.shell() + elif args.main_command == "reproduce": + all_tasks = list(runner.platform["tasks"].keys()) + if args.until not in all_tasks: + print(f"Error: Unknown task '{args.until}'. Available tasks: {', '.join(all_tasks)}", file=sys.stderr) + sys.exit(1) + run_tasks = all_tasks[: all_tasks.index(args.until) + 1] + + if platform_name: + # In quick mode, shorten the message + print(f"[{platform_name}] Running tasks: {', '.join(run_tasks)}") + else: + print("Preparing a new virtual environment with PCP preinstalled, this will take about 20 minutes...") + + started = datetime.now() + runner.setup(args.pcp_path) + for task in run_tasks: + print(f"\n[{platform_name if platform_name else 'CI'}] Running task {task}...") + runner.task(task) + duration_min = (datetime.now() - started).total_seconds() / 60 + print(f"\n[{platform_name if platform_name else 'CI'}] Tasks completed, took {duration_min:.0f}m.") + + if not platform_name: # Only show in non-quick mode + if "install" in all_tasks and all_tasks.index(args.until) >= all_tasks.index("install"): + print("\nPlease run:\n") + print(" sudo -u pcpqa -i ./check XXX\n") + print("to run a QA test. PCP is already installed, from sources located in './pcp'.") + print("Starting a shell in the new virtual environment...\n") + runner.shell() + else: + print(f"Error: Unknown command {args.main_command}", file=sys.stderr) + sys.exit(1) + except subprocess.CalledProcessError as e: + print(f"Error on {platform_name or 'command'}: {e}", file=sys.stderr) + # In quick mode, continue to next platform instead of exiting + if platform_name: + return + sys.exit(1) + + class DirectRunner: def __init__(self, platform_name: str, platform): self.platform_name = platform_name @@ -105,44 +254,62 @@ def get_artifacts(self, artifact, path): class ContainerRunner: - def __init__(self, platform_name: str, platform): + def _setup_macos_config(self): + """Configure podman for macOS (no sudo, platform flags for ARM64).""" + self.sudo = [] + self.security_opts = [] + if is_arm64(): + # Specify container architecture explicitly on ARM64 macOS + arch = "linux/arm64" if self.use_native_arch else "linux/amd64" + self.platform_flags = ["--platform", arch] + + def _setup_linux_config(self): + """Configure podman for Linux (sudo for Ubuntu, system labels).""" + try: + with open("/etc/os-release", encoding="utf-8") as f: + for line in f: + k, v = line.rstrip().split("=") + if k == "NAME" and v == '"Ubuntu"': + self.sudo = ["sudo", "-E", "XDG_RUNTIME_DIR="] + self.security_opts = ["--security-opt", "label=disable"] + break + except FileNotFoundError: + pass + + def __init__(self, platform_name: str, platform, use_native_arch: bool = False): self.platform_name = platform_name self.platform = platform self.container_name = f"pcp-ci-{self.platform_name}" self.image_name = f"{self.container_name}-image" self.command_preamble = "set -eux\nexport runner=container\n" - - # on Ubuntu, systemd inside the container only works with sudo - # also don't run as root in general on Github actions, - # otherwise the direct runner would run everything as root + self.platform_flags = [] + self.use_native_arch = use_native_arch self.sudo = [] self.security_opts = [] - with open("/etc/os-release", encoding="utf-8") as f: - for line in f: - k, v = line.rstrip().split("=") - if k == "NAME": - if v == '"Ubuntu"': - self.sudo = ["sudo", "-E", "XDG_RUNTIME_DIR="] - self.security_opts = ["--security-opt", "label=disable"] - break + + if is_macos(): + self._setup_macos_config() + else: + self._setup_linux_config() def setup(self, pcp_path): containerfile = self.platform["container"]["containerfile"] - # build a new image + # platform_flags specifies container architecture (e.g., --platform linux/arm64) + # on ARM64 macOS, allowing explicit control over native vs emulated builds subprocess.run( - [*self.sudo, "podman", "build", "--squash", "-t", self.image_name, "-f", "-"], + [*self.sudo, "podman", "build", *self.platform_flags, "--squash", "-t", self.image_name, "-f", "-"], input=containerfile.encode(), check=True, ) - # start a new container subprocess.run([*self.sudo, "podman", "rm", "-f", self.container_name], stderr=subprocess.DEVNULL, check=False) subprocess.run( [ *self.sudo, "podman", "run", + *self.platform_flags, "-dt", "--name", self.container_name, @@ -153,9 +320,12 @@ def setup(self, pcp_path): check=True, ) + # Copy PCP sources including .git directory + # Makepkgs requires a valid git repository for status, checkout, and archive operations subprocess.run( - [*self.sudo, "podman", "cp", f"{pcp_path}/", f"{self.container_name}:/home/pcpbuild/pcp"], check=True + [*self.sudo, "podman", "cp", f"{pcp_path}/.", f"{self.container_name}:/home/pcpbuild/pcp"], check=True ) + self.exec("sudo chown -R pcpbuild:pcpbuild .") self.exec("mkdir -p ../artifacts/build ../artifacts/test") @@ -212,10 +382,65 @@ def get_artifacts(self, artifact, path): ) +def _determine_use_native_arch(args): + """Determine whether to use native architecture based on platform and flags.""" + use_native_arch = False + if is_macos(): + # On macOS: default to native arch unless --emulate is specified + use_native_arch = not args.emulate + # On Linux or with --native-arch flag: respect explicit --native-arch + if args.native_arch: + use_native_arch = True + return use_native_arch + + +def _create_runner(platform_name, platform, use_native_arch): + """Create the appropriate runner for the given platform type.""" + platform_type = platform.get("type") + if platform_type == "direct": + return DirectRunner(platform_name, platform) + elif platform_type == "vm": + return VirtualMachineRunner(platform_name, platform) + elif platform_type == "container": + return ContainerRunner(platform_name, platform, use_native_arch=use_native_arch) + else: + print(f"Error: Unknown platform type: {platform_type}", file=sys.stderr) + sys.exit(1) + + +def _load_platform_definition(platform_name): + """Load platform YAML definition file.""" + platform_def_path = os.path.join(os.path.dirname(__file__), f"platforms/{platform_name}.yml") + try: + with open(platform_def_path, encoding="utf-8") as f: + return yaml.safe_load(f) + except FileNotFoundError: + print(f"Error: Platform definition not found: {platform_def_path}", file=sys.stderr) + sys.exit(1) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("--pcp_path", default=".") - parser.add_argument("platform") + parser.add_argument( + "--native-arch", + action="store_true", + help="Use native host architecture instead of amd64 (useful on macOS for faster builds)" + ) + parser.add_argument( + "--emulate", + action="store_true", + help="Force amd64 emulation even on native ARM64 systems (default on non-macOS)" + ) + parser.add_argument( + "--quick", + nargs="?", + const=True, + metavar="PLATFORMS", + help="Quick mode: run multiple platforms. Platforms can be comma or space-separated, " + "or loaded from PCP_CI_QUICK_PLATFORMS env var or .pcp-ci-quick config file" + ) + parser.add_argument("platform", nargs="?") subparsers = parser.add_subparsers(dest="main_command") subparsers.add_parser("setup") @@ -237,66 +462,46 @@ def main(): parser_reproduce.add_argument("--until", default="init_qa") args = parser.parse_args() - platform_def_path = os.path.join(os.path.dirname(__file__), f"platforms/{args.platform}.yml") - with open(platform_def_path, encoding="utf-8") as f: - platform = yaml.safe_load(f) - platform_type = platform.get("type") - if platform_type == "direct": - runner = DirectRunner(args.platform, platform) - elif platform_type == "vm": - runner = VirtualMachineRunner(args.platform, platform) - elif platform_type == "container": - runner = ContainerRunner(args.platform, platform) - if args.main_command == "setup": - try: - runner.setup(args.pcp_path) - runner.task("setup") - except subprocess.CalledProcessError as e: - print(f"Error: {e}", file=sys.stderr) - sys.exit(1) - elif args.main_command == "destroy": - try: - runner.destroy() - except subprocess.CalledProcessError as e: - print(f"Error: {e}", file=sys.stderr) + # Handle quick mode + if args.quick is not None: + quick_platforms = resolve_platforms_to_run( + cli_platforms=args.platform if args.quick is True else args.quick, + env_platforms=os.environ.get("PCP_CI_QUICK_PLATFORMS") + ) + + if not quick_platforms: + print_quick_mode_help() sys.exit(1) - elif args.main_command == "task": - try: - runner.task(args.task_name) - except subprocess.CalledProcessError as e: - print(f"Error: {e}", file=sys.stderr) + + if not args.main_command: + print("Error: Quick mode requires a subcommand (setup, task, reproduce, etc.)", file=sys.stderr) sys.exit(1) - elif args.main_command == "artifacts": - runner.task(f"copy_{args.artifact}_artifacts") - runner.get_artifacts(args.artifact, args.path) - elif args.main_command == "exec": - runner.exec(" ".join(args.command), check=False) - elif args.main_command == "shell": - runner.shell() - elif args.main_command == "reproduce": - all_tasks = ["setup", "build", "install", "init_qa", "qa"] - run_tasks = all_tasks[: all_tasks.index(args.until) + 1] - - print("Preparing a new virtual environment with PCP preinstalled, this will take about 20 minutes...") - started = datetime.now() - runner.setup(args.pcp_path) - for task in run_tasks: - print(f"\nRunning task {task}...") - runner.task(task) - duration_min = (datetime.now() - started).total_seconds() / 60 - print(f"\nVirtual environment setup done, took {duration_min:.0f}m.") - - if all_tasks.index(args.until) >= all_tasks.index("install"): - print("\nPlease run:\n") - print(" sudo -u pcpqa -i ./check XXX\n") - print("to run a QA test. PCP is already installed, from sources located in './pcp'.") - print("Starting a shell in the new virtual environment...\n") - runner.shell() - else: - parser.print_help() + + use_native_arch = _determine_use_native_arch(args) + + for platform_name in quick_platforms: + print(f"\n{'='*60}") + print(f"Running on platform: {platform_name}") + print(f"{'='*60}\n") + + platform = _load_platform_definition(platform_name) + runner = _create_runner(platform_name, platform, use_native_arch) + _execute_command(runner, args, platform_name) + + sys.exit(0) + + # Normal (non-quick) mode + if not args.platform: + print("Error: Platform argument required (unless using --quick mode)", file=sys.stderr) sys.exit(1) + platform = _load_platform_definition(args.platform) + use_native_arch = _determine_use_native_arch(args) + runner = _create_runner(args.platform, platform, use_native_arch) + + _execute_command(runner, args) + if __name__ == "__main__": main() diff --git a/qa/admin/package-lists/Debian+12+aarch64 b/qa/admin/package-lists/Debian+12+aarch64 index 723e2e6520b..9b0e0034778 100644 --- a/qa/admin/package-lists/Debian+12+aarch64 +++ b/qa/admin/package-lists/Debian+12+aarch64 @@ -130,7 +130,7 @@ sed smartmontools socat sysstat -systemd-dev +systemd targetcli-fb time unbound diff --git a/qa/admin/package-lists/Ubuntu+24.04+any b/qa/admin/package-lists/Ubuntu+24.04+any new file mode 100644 index 00000000000..d9b3a240120 --- /dev/null +++ b/qa/admin/package-lists/Ubuntu+24.04+any @@ -0,0 +1,153 @@ +# PCP required package list for Ubuntu 22.04 (any) +# +Text::CSV_XS cpan +apache2-bin +auditd not4ci +autoconf +autotools-dev +avahi-utils +bash +bc +bind9-host +bison +bpfcc-tools +bpftrace +# see https://wiki.ubuntu.com/Debug%20Symbol%20Packages to set up repo +# for this one ... +bpftrace-dbgsym not4ci +bsd-mailx +build-essential +chrpath +clang +coreutils +cppcheck +cron +curl +debhelper +dh-python +docker.io not4ci +dpkg-dev +ed +ethtool +expect +flex +g++ +gawk +gcc +gdb +gfs2-utils +git +grep +iproute2 +jq +libavahi-common-dev +libbpf-dev +libbpf1 +libclass-dbi-perl +libcmocka-dev +libcoin-dev +libdbd-mysql-perl +libdbd-pg-perl +libdevmapper-dev +libdrm-dev +libextutils-autoinstall-perl +libfile-slurp-perl +libgl1-mesa-dri +libibmad-dev +libibumad-dev +libibverbs-dev +libicu74 +libinih-dev +libjson-perl +liblist-moreutils-perl +liblzma-dev +libncurses-dev +libnet-snmp-perl +libperl5.38t64 +libpfm4-dev +libpython3-dev +libpython3-stdlib +libqt5svg5-dev +libreadline-dev +librrds-perl +libsasl2-dev +libsasl2-modules +libsoqt520-dev +libspreadsheet-read-perl +libspreadsheet-readsxc-perl +libspreadsheet-writeexcel-perl +libspreadsheet-xlsx-perl +libsqlite3-0 +libssl-dev +libsystemd-dev +libtext-csv-xs-perl +libtimedate-perl +libuv1-dev +libvirt-daemon +libvirt-daemon-system +libxml-libxml-perl +libxml-tokeparser-perl +libyaml-libyaml-perl +linux-headers-`uname -r` not4ci +llvm +lm-sensors +make +man-db +mandoc +mariadb-client-core +memcached +net-tools +nmap +openjdk-21-jre-headless +openssl +perl +perl-modules-5.38 +# perl-xs-dev is a virtual package and libperl-dev is an alias at least up to +# Debian 13 (trixie) +libperl-dev +pkg-config +postgresql-client-common +psmisc +pylint +python3-all +python3-all-dev +python3-bpfcc +python3-dev +python3-elasticsearch +python3-libvirt +python3-lxml +python3-minimal +python3-openpyxl +python3-pandas +python3-pil +python3-prometheus-client +python3-psycopg2 +python3-pymongo +python3-pyodbc +python3-requests +python3-setuptools +python3-six +qtbase5-dev +qtbase5-dev-tools +qtchooser +redis-redisearch +redis-server +redis-tools +sasl2-bin +sed +smartmontools +socat +sudo +sysstat +systemd-dev +targetcli-fb +time +unbound +valkey-server +valkey-tools +valgrind +xfsprogs +xkb-data +zfsutils-linux +zlib1g-dev +zstd