diff --git a/.devcontainer/setup.py b/.devcontainer/setup.py index c68ca08fa1a..69b62c7ad98 100755 --- a/.devcontainer/setup.py +++ b/.devcontainer/setup.py @@ -77,7 +77,10 @@ def setup(): # Turns fsync off. Create database operations with fsync on are very slow on Ubuntu. # Having fsync off in dev environment is fine. "sed -i -e \"s/#fsync = on/fsync = off/\" /srv/postgres14/postgresql.conf", + # Configure pg_hba.conf for password authentication from all hosts (dev environment only) + # Note: In dev, we allow both trust and scram-sha-256 for convenience "echo 'host all all 0.0.0.0/0 trust' >> /srv/postgres14/pg_hba.conf", + "echo 'host all all 0.0.0.0/0 scram-sha-256' >> /srv/postgres14/pg_hba.conf", # "supervisorctl restart postgresql", ]) diff --git a/.github/workflows/build-pipeline.yml b/.github/workflows/build-pipeline.yml new file mode 100644 index 00000000000..f149524f3d9 --- /dev/null +++ b/.github/workflows/build-pipeline.yml @@ -0,0 +1,34 @@ +name: Build Pipeline + +on: + workflow_dispatch: + inputs: + target: + description: 'Build target: all | client | server' + required: true + default: 'all' + type: choice + options: + - all + - client + - server + pr_number: + description: 'PR number (informational — used for logging)' + required: false + default: '' + +jobs: + build: + name: Build PMM (${{ github.event.inputs.target }}) + # Requires a self-hosted runner with Docker + Minio configured. + # Adjust the runner label once the build infrastructure is in place. + runs-on: [self-hosted, pmm-builder] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build + run: | + cd build/pipeline + make build-${{ github.event.inputs.target }} diff --git a/.github/workflows/pmm-bot.yml b/.github/workflows/pmm-bot.yml new file mode 100644 index 00000000000..9f21b4f8764 --- /dev/null +++ b/.github/workflows/pmm-bot.yml @@ -0,0 +1,111 @@ +name: PMM Bot + +on: + issue_comment: + types: [created] + +jobs: + dispatch: + # Only handle PR comments that start with @pmm-bot + if: > + github.event.issue.pull_request != null && + startsWith(github.event.comment.body, '@pmm-bot') + runs-on: ubuntu-latest + permissions: + issues: write # post comments and reactions + pull-requests: write + actions: write # trigger workflow_dispatch + + steps: + - uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const body = context.payload.comment.body.trim(); + const sender = context.payload.comment.user.login; + const owner = context.repo.owner; + const repo = context.repo.repo; + const issueNumber = context.payload.issue.number; + const commentId = context.payload.comment.id; + + // ── 1. Permission check ────────────────────────────────────────── + let permission; + try { + const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner, repo, username: sender, + }); + permission = data.permission; + } catch (e) { + // Non-collaborators get a 404 from this endpoint. + permission = 'none'; + } + + if (!['write', 'admin'].includes(permission)) { + await github.rest.issues.createComment({ + owner, repo, issue_number: issueNumber, + body: `@${sender} You need write access to trigger bot commands.`, + }); + return; + } + + // ── 2. Acknowledge ─────────────────────────────────────────────── + await github.rest.reactions.createForIssueComment({ + owner, repo, comment_id: commentId, content: 'eyes', + }); + + // ── 3. Resolve the PR branch ───────────────────────────────────── + const { data: pr } = await github.rest.pulls.get({ + owner, repo, pull_number: issueNumber, + }); + const ref = pr.head.ref; + + // ── 4. Parse command ───────────────────────────────────────────── + // Strip the "@pmm-bot " prefix and normalise whitespace. + const cmd = body.replace(/^@pmm-bot\s+/i, '').trim().toLowerCase(); + + const COMMANDS = { + 'rebuild all': { workflow: 'build-pipeline.yml', inputs: { target: 'all' } }, + 'rebuild client': { workflow: 'build-pipeline.yml', inputs: { target: 'client' } }, + 'rebuild server': { workflow: 'build-pipeline.yml', inputs: { target: 'server' } }, + 'run api tests': { workflow: 'api-tests.yml', inputs: {} }, + }; + + const HELP_TEXT = + '**Available commands:**\n' + + '- `@pmm-bot rebuild all` — build client + server\n' + + '- `@pmm-bot rebuild client` — build client only\n' + + '- `@pmm-bot rebuild server` — build server only\n' + + '- `@pmm-bot run API tests` — run API test suite\n' + + '- `@pmm-bot help` — show this message\n'; + + if (cmd === 'help') { + await github.rest.issues.createComment({ + owner, repo, issue_number: issueNumber, body: HELP_TEXT, + }); + return; + } + + if (!(cmd in COMMANDS)) { + await github.rest.issues.createComment({ + owner, repo, issue_number: issueNumber, + body: `Unknown command: \`${cmd}\`\n\n${HELP_TEXT}`, + }); + return; + } + + // ── 5. Dispatch ────────────────────────────────────────────────── + const { workflow, inputs } = COMMANDS[cmd]; + await github.rest.actions.createWorkflowDispatch({ + owner, repo, + workflow_id: workflow, + ref, + inputs: { ...inputs, pr_number: String(issueNumber) }, + }); + + // ── 6. Confirm ─────────────────────────────────────────────────── + const runsUrl = + `https://github.com/${owner}/${repo}/actions/workflows/${workflow}`; + await github.rest.issues.createComment({ + owner, repo, issue_number: issueNumber, + body: `@${sender} Dispatched \`${cmd}\` on branch \`${ref}\`. [View runs](${runsUrl})`, + }); diff --git a/.gitignore b/.gitignore index 503c3b8e270..c52e9a865f5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /.vscode/ *.iml +# Temprorary build files /api/nginx/*.pem bin/ !/documentation/resources/bin/ @@ -13,7 +14,6 @@ cover.out crosscover.out agent/testdata/ - agent/agents/mysql/slowlog/parser/corpus/ agent/agents/mysql/slowlog/parser/crashers/ agent/agents/mysql/slowlog/parser/suppressions/ @@ -32,22 +32,22 @@ compose.yml # ViM temporary files *.sw[o,p] +# build ops .env .netrc .modules build.log +packer.log ci.yml -api-tests/pmm-api-tests-output.txt -api-tests/pmm-api-tests-junit-report.xml +pmm-api-tests-output.txt +pmm-api-tests-junit-report.xml -packer.log encryption.key /tmp/ settings.json GEMINI.md -RTA-Jira-Tasks-Summary.md # PMM Demo backups (it would increase repository size significantly) dev/clickhouse-backups/ diff --git a/build/ansible/roles/pmm-images/tasks/main.yml b/build/ansible/roles/pmm-images/tasks/main.yml index 954f10817f4..40bc3a0e9b7 100644 --- a/build/ansible/roles/pmm-images/tasks/main.yml +++ b/build/ansible/roles/pmm-images/tasks/main.yml @@ -10,15 +10,6 @@ - name: List installed gpg keys command: ls -la /etc/pki/rpm-gpg -# Local yum repo for building pmm server docker image in autobuild jobs -- name: Add a local YUM repository - yum_repository: - name: local - description: Local YUM repository - x86_64 - baseurl: file:///tmp/RPMS - gpgcheck: no - enabled: no - - name: Update OS packages dnf: name: "*" @@ -96,18 +87,110 @@ - /var/lib/cloud/scripts/per-once - /var/lib/cloud/scripts/per-boot -- name: Install PMM Server components - dnf: - name: - - percona-grafana - - percona-victoriametrics - - percona-qan-api2 - - percona-dashboards - - pmm-managed - - pmm-dump - - vmproxy - state: installed - enablerepo: local +- name: Create server component directories + file: + path: "{{ item }}" + state: directory + owner: pmm + group: root + mode: 0775 + loop: + - /usr/share/grafana + - /usr/share/pmm-managed + - /usr/share/pmm-ui + - /usr/share/percona-dashboards + - /usr/share/percona-dashboards/panels + - /usr/local/percona + - /usr/local/percona/advisors + - /usr/local/percona/checks + - /usr/local/percona/alerting-templates + - /var/lib/grafana + +- name: Install Go binaries + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: "0755" + remote_src: yes + loop: + - { src: /opt/pmm-staging/pmm-managed/bin/pmm-managed, dest: /usr/sbin/pmm-managed } + - { src: /opt/pmm-staging/pmm-managed/bin/pmm-encryption-rotation, dest: /usr/sbin/pmm-encryption-rotation } + - { src: /opt/pmm-staging/pmm-managed/bin/pmm-managed-init, dest: /usr/sbin/pmm-managed-init } + - { src: /opt/pmm-staging/pmm-managed/bin/pmm-managed-starlark, dest: /usr/sbin/pmm-managed-starlark } + - { src: /opt/pmm-staging/pmm-managed/bin/qan-api2, dest: /usr/sbin/percona-qan-api2 } + - { src: /opt/pmm-staging/pmm-managed/bin/vmproxy, dest: /usr/sbin/vmproxy } + - { src: /opt/pmm-staging/pmm-dump/pmm-dump, dest: /usr/sbin/pmm-dump } + - { src: /opt/pmm-staging/grafana-go/grafana-server, dest: /usr/sbin/grafana-server } + - { src: /opt/pmm-staging/grafana-go/grafana, dest: /usr/sbin/grafana } + - { src: /opt/pmm-staging/grafana-go/grafana-cli, dest: /usr/bin/grafana-cli } + - { src: /opt/pmm-staging/victoriametrics/victoria-metrics-pure, dest: /usr/sbin/victoriametrics } + - { src: /opt/pmm-staging/victoriametrics/vmalert-pure, dest: /usr/sbin/vmalert } + +- name: Install Grafana UI assets + command: cp -rT /opt/pmm-staging/grafana-ui/{{ item.src }} /usr/share/grafana/{{ item.dest }} + loop: + - { src: public, dest: public } + - { src: conf, dest: conf } + - { src: tools, dest: tools } + +- name: Install Grafana configuration files + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: pmm + group: root + mode: "0644" + remote_src: yes + loop: + - { src: /opt/pmm-staging/grafana-ui/conf/sample.ini, dest: /etc/grafana/grafana.ini } + - { src: /opt/pmm-staging/grafana-ui/conf/ldap.toml, dest: /etc/grafana/ldap.toml } + +- name: Install pmm-managed data assets + command: cp -rT /opt/pmm-staging/pmm-managed/data/{{ item.src }} {{ item.dest }} + loop: + - { src: swagger, dest: /usr/share/pmm-managed/swagger } + - { src: advisors, dest: /usr/local/percona/advisors } + - { src: checks, dest: /usr/local/percona/checks } + - { src: alerting-templates, dest: /usr/local/percona/alerting-templates } + +- name: Install PMM UI assets + command: cp -rT /opt/pmm-staging/pmm-ui/{{ item.src }} {{ item.dest }} + loop: + - { src: pmm-dist, dest: /usr/share/pmm-ui } + - { src: pmm-compat-dist, dest: /usr/share/percona-dashboards/panels/pmm-compat-app } + +- name: Install percona-dashboards assets + command: cp -rT /opt/pmm-staging/pmm-dashboards/{{ item.src }} {{ item.dest }} + loop: + - { src: panels, dest: /usr/share/percona-dashboards/panels } + - { src: pmm-app-dist, dest: /usr/share/percona-dashboards/panels/pmm-app } + +- name: Install VERSION.json + copy: + src: /opt/pmm-staging/version.json + dest: /usr/share/pmm-server/VERSION.json + owner: root + group: root + mode: "0644" + remote_src: yes + +- name: Set ownership on pmm-owned assets + file: + path: "{{ item }}" + state: directory + owner: pmm + group: root + recurse: yes + loop: + - /usr/share/grafana + - /usr/share/pmm-managed + - /usr/share/pmm-ui + - /usr/share/percona-dashboards + - /usr/local/percona + + - name: Configure grafana include_role: diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9 index c6a9e539057..53f4d2ddc66 100644 --- a/build/docker/server/Dockerfile.el9 +++ b/build/docker/server/Dockerfile.el9 @@ -19,7 +19,8 @@ RUN --mount=type=cache,target=/var/cache/dnf \ ansible-collection-ansible-posix \ glibc-langpack-en \ dnf \ - vi + vi && \ + dnf -y remove microdnf COPY entrypoint.sh /opt/entrypoint.sh COPY ansible /opt/ansible diff --git a/build/pipeline/.env.example b/build/pipeline/.env.example new file mode 100644 index 00000000000..c8577cf1bae --- /dev/null +++ b/build/pipeline/.env.example @@ -0,0 +1,68 @@ +# PMM Build Environment Variables +# Copy this file to .env and run 'scripts/migrate-from-submodules' to fill in +# values from percona/pmm-submodules, or populate manually. +# +# Format: KEY=VALUE (no spaces around '=', no inline comments) + +# Product version +PMM_VERSION= + +# ── Server components ──────────────────────────────────────────────────────── + +PMM_URL=https://github.com/percona/pmm.git +PMM_REF= +PMM_UI_REF= +PMM_DASHBOARDS_REF= + +GRAFANA_URL=https://github.com/percona/grafana.git +GRAFANA_REF= + +VM_URL=https://github.com/VictoriaMetrics/VictoriaMetrics.git +VM_REF= + +PMM_DUMP_URL=https://github.com/percona/pmm-dump.git +PMM_DUMP_REF= + +# ── Client components ──────────────────────────────────────────────────────── + +NODE_EXPORTER_URL=https://github.com/percona/node_exporter.git +NODE_EXPORTER_REF= + +MYSQLD_EXPORTER_URL=https://github.com/percona/mysqld_exporter.git +MYSQLD_EXPORTER_REF= + +MONGODB_EXPORTER_URL=https://github.com/percona/mongodb_exporter.git +MONGODB_EXPORTER_REF= + +POSTGRES_EXPORTER_URL=https://github.com/percona/postgres_exporter.git +POSTGRES_EXPORTER_REF= + +PROXYSQL_EXPORTER_URL=https://github.com/percona/proxysql_exporter.git +PROXYSQL_EXPORTER_REF= + +RDS_EXPORTER_URL=https://github.com/percona/rds_exporter.git +RDS_EXPORTER_REF= + +AZURE_METRICS_EXPORTER_URL=https://github.com/percona/azure_metrics_exporter.git +AZURE_METRICS_EXPORTER_REF= + +REDIS_EXPORTER_URL=https://github.com/oliver006/redis_exporter.git +REDIS_EXPORTER_REF= + +VMAGENT_URL=https://github.com/VictoriaMetrics/VictoriaMetrics.git +VMAGENT_REF= + +NOMAD_URL=https://github.com/hashicorp/nomad.git +NOMAD_REF= + +PERCONA_TOOLKIT_URL=https://github.com/percona/percona-toolkit.git +PERCONA_TOOLKIT_REF= + +# ── Minio S3 cache ─────────────────────────────────────────────────────────── +# MINIO_ENDPOINT is auto-detected (host.docker.internal on macOS, Docker bridge +# gateway on Linux). Override only if MinIO runs on a non-standard host/port: +# MINIO_ENDPOINT=http://192.168.1.10:9000 +MINIO_BUCKET=cache +MINIO_CACHE_PREFIX=repos +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= diff --git a/build/pipeline/.gitignore b/build/pipeline/.gitignore new file mode 100644 index 00000000000..aef37db64b0 --- /dev/null +++ b/build/pipeline/.gitignore @@ -0,0 +1,13 @@ +# Build outputs +output/ +package/ + +# Local cache directory (synced from Minio) +.cache/ + +# Temporary build files +pmm-client.tar.gz +gitCommit + +# Build log +build.log diff --git a/build/pipeline/AGENT.md b/build/pipeline/AGENT.md new file mode 100644 index 00000000000..e65afb4bc02 --- /dev/null +++ b/build/pipeline/AGENT.md @@ -0,0 +1,440 @@ +# PMM Build Pipeline - AI Agent Guidelines + +This document provides comprehensive guidelines for AI agents working with the PMM build pipeline. The build pipeline is a Docker-based system for building PMM components. + +## Architecture Overview + +The build pipeline uses `docker run` for all server component builds, outputting artifacts directly to the host via Docker volumes. Client components are built with a custom `pmm-builder` image. Different images are used for different server components depending on their requirements. + +### Core Components + +1. **Dockerfile.builder** - Defines the `pmm-builder` image (golang-based) for Go component builds +2. **Dockerfile.server** - Assembly-only Dockerfile; copies pre-built artifacts from host paths +3. **scripts/build-component** - Main build orchestration script for client builds +4. **.env** - Single source of truth for all component URLs, refs, and PMM_VERSION +5. **Makefile** - Build targets and convenience commands +6. **README.md** - User-facing documentation + +> **Note:** All server components are built via `docker run` (no component Dockerfiles). Each +> `build-artifact-*` target in the Makefile runs a container, clones the source from a local +> bare-repo cache, builds, and copies artifacts to `output/server//` on the host. +> Dockerfile.server then assembles the runtime image from those host-side paths. + +### Design Principles + +- **Volume Caching** - Use Docker volumes for Go modules and build artifacts +- **Single Source of Truth** - All component URLs, git refs, and `PMM_VERSION` live in `.env`. Run `scripts/migrate-from-submodules` once to populate from percona/pmm-submodules. `GO_VERSION` is defined once in Makefile and passed as `--build-arg` to all component Dockerfiles +- **Explicit REF args** - All `*_REF` build args in component Dockerfiles have no defaults; they must be passed via `--build-arg`. Omitting one causes an immediate build failure at `git checkout` +- **Split server builds** - All server components are built independently via `docker run`, writing artifacts to `output/server//` on the host: + - **pmm-managed, pmm-dump, VictoriaMetrics**: `pmm-builder:latest` (pure Go, `CGO_ENABLED=0`), run at `HOST_ARCH` for native speed; Go cross-compiles for `GOARCH` + - **grafana-go**: `golang:$(GO_VERSION)` (CGO enabled, has gcc for `go-sqlite3`), runs at `--platform linux/$(GOARCH)` so sqlite3's C code compiles natively for the target arch + - **grafana-ui, pmm-dashboards, pmm-ui**: `node:22` (git included), run at `HOST_ARCH`; Yarn cache at `/usr/local/share/.cache/yarn` shared via `YARN_CACHE_VOL` + - **BuildKit** (`docker buildx build`) is used **only** for the final `build-server-docker` step (OracleLinux runtime image assembly with S3 layer cache) + - Node components build sequentially to avoid Yarn network saturation +- **Platform Awareness** - Explicit --platform flags to avoid warnings +- **Minimal Containers** - Run as root in golang image, no permission issues + +## Key Files and Their Roles + +### Dockerfile.builder + +```dockerfile +ARG GO_VERSION=latest +FROM golang:${GO_VERSION} +``` + +**Purpose**: Define the build environment +**Key Points**: +- Based on official golang image (currently uses `latest` by default) +- Installs build dependencies: zip (for nomad), and some other tools (like gssapi-dev for dynamic builds) +- Sets default Go environment variables +- Runs as root (no user permission issues) + +**When to modify**: +- Adding new build tools needed by components +- Changing base Go version (via GO_VERSION build arg) + +### scripts/build-component + +**Purpose**: Main build orchestration +**Key Functions**: +- `build_builder_image()` - Ensures pmm-builder image exists +- `create_volumes()` - Creates Docker volumes for caching +- `build_workspace_component()` - Builds pmm-admin/pmm-agent from the monorepo +- `build_external_component()` - Clones and builds external components using `*_URL` / `*_REF` from `.env` + +**Key Variables**: +```bash +BUILDER_IMAGE="pmm-builder:latest" +GOMOD_CACHE_VOL="pmm-mod" # Go module cache +BUILD_CACHE_VOL="pmm-build" # Build artifacts cache +PLATFORM="${PLATFORM:-linux/amd64}" +``` + +**When to modify**: +- Adding new workspace components (update case statements) +- Adding new external components (update component lists and build commands) +- Changing Docker volume paths +- Modifying build environment variables + +### scripts/package-tarball + +**Purpose**: Generate pmm-client distribution tarball +**Output**: `pmm-client-${VERSION}.tar.gz` in `PACKAGE_DIR` +**Dependencies**: Requires built components in `OUTPUT_DIR` + +**Archive Structure**: +``` +pmm-client-${VERSION}/ +├── bin/ # All built binaries +├── config/ # systemd service files +├── debian/ # Debian packaging files +├── rpm/ # RPM spec files +├── queries-*.yml # Query examples (if present) +├── install_tarball # Installation script +└── VERSION # Version identifier +``` + +**Key Variables**: +```bash +OUTPUT_DIR="${OUTPUT_DIR:-./output}" # Source binaries +PACKAGE_DIR="${PACKAGE_DIR:-./package}" # Output location +PMM_VERSION="${PMM_VERSION}" # Version string +``` + +**When to modify**: +- Adding new files to distribution +- Changing directory structure +- Modifying VERSION file format +- Adjusting query file locations + +### Makefile + +**Purpose**: User-facing build targets +**Key Targets**: +- `build` - Build single component (requires COMPONENT=) +- `build-all` - Build all components sequentially +- `build-dynamic` - Build with dynamic linking (GSSAPI) +- `build-arm64` - Build for ARM64 architecture +- `builder-image` - Build pmm-builder Docker image +- `package-tarball` - Generate pmm-client distribution archive +- `clean` - Remove output directory +- `clean-volumes` - Remove Docker volumes (cache) + +**Key Variables**: +```makefile +WORKSPACE_COMPONENTS := pmm-admin pmm-agent +EXTERNAL_COMPONENTS := node_exporter mysqld_exporter ... +GO_VERSION ?= 1.26 +HOST_ARCH := $(shell uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') # native host arch +GOARCH ?= amd64 # target architecture for compiled binaries +PLATFORM ?= linux/amd64 # used by client builds +GOMOD_CACHE_VOL ?= pmm-mod # Docker volume: Go module cache +BUILD_CACHE_VOL ?= pmm-build # Docker volume: Go build cache +YARN_CACHE_VOL ?= pmm-yarn # Docker volume: Yarn package cache +SERVER_OUTPUT_DIR := $(PIPELINE_DIR)output/server +OUTPUT_DIR ?= ./output +PACKAGE_DIR ?= ./package +``` + +**When to modify**: +- Adding new components (update component lists) +- Adding new build targets +- Changing default values + +## Common Development Patterns + +### Adding a New Workspace Component + +1. Add to `WORKSPACE_COMPONENTS` in Makefile: +```makefile +WORKSPACE_COMPONENTS := pmm-admin pmm-agent new-component +``` + +2. Add to component list in `build-component`: +```bash +WORKSPACE_COMPONENTS="pmm-admin pmm-agent new-component" +``` + +3. Add subdirectory mapping in `build_workspace_component()`: +```bash +case "${component}" in + pmm-admin) subdir="admin" ;; + pmm-agent) subdir="agent" ;; + new-component) subdir="newcomponent" ;; +``` + +4. Add build command in `build_workspace_component()`: +```bash +case "${component}" in + new-component) + build_cmd="make -C /workspace/${subdir} release PMM_RELEASE_PATH=/output" + ;; +``` + +### Adding a New External Component + +1. Add to `EXTERNAL_COMPONENTS` in both Makefile and `build-component` + +2. Add `NEW_EXPORTER_URL` and `NEW_EXPORTER_REF` to `.env` (and `.env.example`). + +3. Add build command in `build_external_component()`: +```bash +case "${component}" in + new_exporter) + build_cmd="make build && cp new_exporter /output/" + ;; +``` + +### Modifying Docker Volumes + +Volume paths are critical for caching: + +**Both workspace and external components**: +```bash +-v "${GOMOD_CACHE_VOL}:/go/pkg/mod" \ +-v "${BUILD_CACHE_VOL}:/root/.cache/go-build" \ +-e GOCACHE=/root/.cache/go-build \ +``` + +**External components also mount source cache**: +```bash +-v "${SOURCE_CACHE_VOL}:/build/source" \ +``` + +**Important**: External components clone into `/build/source/${component}` (component-specific subdirectories to avoid conflicts). The volume persists repositories between builds, using `git clean -fdx` to ensure clean state. + +## Critical Conventions + +### Do's + +- **Always use --platform flag** in docker run/build commands +- **Export variables** in Makefile that need to be passed to build-component +- **Keep component lists in sync** between Makefile and build-component script +- **Use realpath** for path resolution (portable across systems) +- **Check for component existence** before building +- **Use case statements** instead of associative arrays (broader shell compatibility) +- **Log informative messages** during build steps +- **Update the README.md** when adding/modifying components or build steps + +### Don'ts + +- **Don't hardcode version numbers** - use `latest` for flexibility +- **Don't use --user flag** - golang image runs as root, no permission issues +- **Don't add runtime network fetches** - all URLs and refs must come from `.env`, not fetched at build time +- **Don't assume volumes are writable** - they're created as root, but golang image handles this +- **Don't use complex shell features** - keep it POSIX-compatible when possible +- **Don't nest Make calls** unnecessarily - use $(call) for reusable functions + +### Error Handling + +```bash +# Always check for required variables +PMM_VERSION="${PMM_VERSION:?No PMM_VERSION specified}" + +# Provide clear error messages +if [ -z "$url" ] || [ -z "$ref" ]; then + echo "Error: Could not determine URL or ref for ${component}" >&2 + return 1 +fi + +# Exit on errors +set -o errexit +set -o pipefail +set -o nounset +``` + +### Build Commands + +**Workspace components** use their native Makefiles: +```bash +make -C /workspace/admin release PMM_RELEASE_PATH=/output +``` + +**External components** follow their own build systems: +```bash +make release && cp binary /output/ +``` + +## Docker Platform Handling + +Always specify platform to avoid warnings on Apple Silicon: + +```bash +docker run --rm \ + --platform "${PLATFORM}" \ + ... + +docker build \ + --platform "${PLATFORM}" \ + ... +``` + +Platform is auto-detected in Makefile but can be overridden: +```bash +PLATFORM=linux/arm64 make build COMPONENT=pmm-admin +``` + +## Caching Strategy + +Four Docker volumes provide caching: + +1. **pmm-mod** (`/go/pkg/mod`) - Go modules, shared across all Go builds +2. **pmm-build** (`/root/.cache/go-build`) - Go build cache, shared across all Go builds +3. **pmm-yarn** (`/usr/local/share/.cache/yarn`) - Yarn package cache for Node.js server builds (grafana-ui, pmm-dashboards, pmm-ui) + +Both server and client builds require bare repos to be present in `REPO_CACHE_DIR` (`.cache/repos/`) — populated from Minio by `make download-cache`. A missing bare repo is a hard failure in both cases; there is no internet-clone fallback like there was in the old build system. + +| Bare repo | Used by | +|-----------|--------| +| `pmm.git` | server (pmm-managed, qan-api2, vmproxy, UI, dashboards) | +| `pmm-dump.git` | server | +| `grafana.git` | server (grafana-go, grafana-ui) | +| `VictoriaMetrics.git` | server + client (vmagent) | +| `node_exporter.git` | client | +| `mysqld_exporter.git` | client | +| `mongodb_exporter.git` | client | +| `postgres_exporter.git` | client | +| `proxysql_exporter.git` | client | +| `rds_exporter.git` | client | +| `azure_metrics_exporter.git` | client | +| `redis_exporter.git` | client | +| `nomad.git` | client | +| `percona-toolkit.git` | client | + +Use `make populate-cache` to clone all missing repos from upstream, then `make update-cache` to push +them to Minio. `make download-cache` syncs the full set from Minio before any build. + +Cache is persistent across builds. Clear with: +```bash +make clean-volumes # Warning: destroys all Go/Yarn caches! +make clean-cache # Warning: removes all bare repos! +``` + +Cache is persistent across builds. Clear with: +```bash +make clean-volumes # Warning: destroys all caches! +``` + +## Component Metadata + +All component URLs and git refs are stored in `.env` as `_URL` and `_REF` pairs +(e.g. `NODE_EXPORTER_URL`, `NODE_EXPORTER_REF`). `PMM_VERSION` is also stored there. + +The `percona/pmm` monorepo contains three independently versioned sub-trees, each with its +own ref variable pointing at the same `pmm.git` bare repo: + +| Variable | Sub-tree | Build target | +|---|---|---| +| `PMM_REF` | Backend (`managed/`, `qan-api2/`, `vmproxy/`) | `build-pmm-managed` | +| `PMM_UI_REF` | Frontend (`ui/`) | `build-pmm-ui` | +| `PMM_DASHBOARDS_REF` | Dashboards (`dashboards/`) | `build-pmm-dashboards` | + +Most of the time all three are the same commit, but separate frontend or dashboards PRs can +set `PMM_UI_REF` / `PMM_DASHBOARDS_REF` to a different branch without touching the backend. + +Run `scripts/migrate-from-submodules` once to populate empty values from the legacy +percona/pmm-submodules repository. After that the build has no network dependency on +pmm-submodules. + +## Make Target Patterns + +### Reusable Functions + +Use Make's `define` for DRY: +```makefile +define check_component + @if [ -z "$(COMPONENT)" ]; then \ + echo "Error: COMPONENT not specified"; \ + exit 1; \ + fi +endef + +build: + $(call check_component) + @$(BUILD_SCRIPT) $(COMPONENT) +``` + +### Target Dependencies + +```makefile +build: volumes # Ensure volumes exist before building +``` + +## Testing Changes + +1. **Build the builder image**: +```bash +make builder-image +``` + +2. **Test single component**: +```bash +make build COMPONENT=pmm-admin +``` + +3. **Verify artifacts**: +```bash +ls -lh output/ +``` + +4. **Test cache persistence**: +```bash +make build COMPONENT=pmm-admin # Should be fast on second run +``` + +## Troubleshooting + +### "PMM_VERSION is not set" + +Either run the migration script or set it directly in `.env`: +```bash +scripts/migrate-from-submodules # fetches from pmm-submodules once +# — or — +echo "PMM_VERSION=3.2.0" >> build/pipeline/.env +``` + +### "NODE_EXPORTER_URL or NODE_EXPORTER_REF is not set in .env" + +The component's URL or ref is missing from `.env`. Run `scripts/migrate-from-submodules` +or add the values manually. + +### Permission denied in volumes + +Should not happen with golang image (runs as root). If it does, check volume mount paths. + +### Platform mismatch warning + +Add `--platform "${PLATFORM}"` to docker command. + +## Integration with CI/CD + +Minimal example: +```yaml +- name: Build Components + run: | + cd build/pipeline + make build-all + env: + PMM_VERSION: ${{ github.ref_name }} +``` + +For specific components: +```bash +make build COMPONENT=pmm-admin +make build COMPONENT=mysqld_exporter +``` + +## Future Enhancements + +When extending the build pipeline: + +1. **Build matrix** - Multiple architectures/build types in one command +2. **Artifact signing** - Add GPG signing step +3. **Image scanning** - Security scan of pmm-builder image +4. **Build metrics** - Timing and size tracking + +## References + +- Main docs: [README.md](README.md) +- Project guidelines: [../../.github/copilot-instructions.md](../../.github/copilot-instructions.md) diff --git a/build/pipeline/Dockerfile.builder b/build/pipeline/Dockerfile.builder new file mode 100644 index 00000000000..88b76dfc9b6 --- /dev/null +++ b/build/pipeline/Dockerfile.builder @@ -0,0 +1,18 @@ +ARG GO_VERSION=latest +FROM golang:${GO_VERSION} + +# Install dependencies needed for building PMM components +RUN apt-get update && \ + apt-get install -y \ + zip \ + libc-dev \ + libkrb5-dev \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +# Set up Go environment +ENV CGO_ENABLED=0 +ENV GOOS=linux +ENV GOARCH=amd64 + +WORKDIR /build diff --git a/build/pipeline/Dockerfile.client b/build/pipeline/Dockerfile.client new file mode 100644 index 00000000000..0081a5d9378 --- /dev/null +++ b/build/pipeline/Dockerfile.client @@ -0,0 +1,49 @@ +FROM oraclelinux:9-slim + +ARG VERSION +ARG RELEASE +ARG BUILD_DATE + +RUN --mount=type=cache,target=/var/cache/dnf \ + microdnf install -y shadow-utils jq \ + && curl -o /tmp/percona-release.rpm https://repo.percona.com/yum/percona-release-latest.noarch.rpm \ + && rpm -i /tmp/percona-release.rpm \ + && rm /tmp/percona-release.rpm \ + && percona-release enable ps-80 \ + && microdnf install -y percona-server-client \ + perl \ + perl-DBI \ + perl-DBD-MySQL \ + perl-Digest-MD5 \ + perl-Time-HiRes \ + perl-IO-Socket-SSL \ + perl-TermReadKey \ + && microdnf clean all + +RUN groupadd -g 1002 pmm-agent && \ + useradd -u 1002 -r -g pmm-agent -s /sbin/nologin \ + -d /usr/local/percona/pmm \ + -c "PMM Client User" pmm-agent + +RUN --mount=type=bind,source=../pmm-client.tar.gz,target=/app/pmm-client.tar.gz \ + tar -xzf /app/pmm-client.tar.gz -C /tmp \ + && cd /tmp/pmm-client* \ + && env PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \ + && cd /tmp \ + && rm -rf /tmp/pmm-client* + +COPY LICENSE /licenses/ + +LABEL org.opencontainers.image.created=${BUILD_DATE} \ + org.opencontainers.image.licenses=Apache-2.0 \ + org.opencontainers.image.title="PMM Client" \ + org.opencontainers.image.description="OCI image for Percona Monitoring and Management Client" \ + org.opencontainers.image.vendor="Percona, LLC" \ + org.opencontainers.image.version=${VERSION} \ + org.opencontainers.image.docs="https://docs.percona.com/percona-monitoring-and-management" + +USER pmm-agent +WORKDIR /usr/local/percona/pmm/ +ENV PATH=/usr/local/percona/pmm/bin/:$PATH + +ENTRYPOINT ["/usr/local/percona/pmm/bin/pmm-agent-entrypoint"] diff --git a/build/pipeline/Dockerfile.server b/build/pipeline/Dockerfile.server new file mode 100644 index 00000000000..7c323816a11 --- /dev/null +++ b/build/pipeline/Dockerfile.server @@ -0,0 +1,57 @@ +# syntax=docker/dockerfile:1 +FROM oraclelinux:9-slim + +ARG VERSION +ARG BUILD_DATE + +ENV LANG=en_US.utf8 +ENV GF_PLUGIN_DIR=/srv/grafana/plugins +ENV PERCONA_TELEMETRY_DISABLE=1 +ENV PS1="[\u@\h \W] # " + +WORKDIR /opt + +# Install system dependencies +RUN --mount=type=cache,target=/var/cache \ + microdnf -y install epel-release && \ + microdnf -y install \ + ansible-core \ + ansible-collection-community-general \ + ansible-collection-community-postgresql \ + ansible-collection-ansible-posix \ + glibc-langpack-en \ + dnf \ + vi && \ + dnf -y remove microdnf + +# Copy Ansible playbooks and run configuration +COPY docker/server/entrypoint.sh /opt/entrypoint.sh +COPY ansible /opt/ansible + +RUN --mount=type=cache,target=/var/cache \ + --mount=type=bind,source=pipeline/pmm-client.tar.gz,target=/tmp/pmm-client.tar.gz \ + --mount=type=bind,source=pipeline/output/server,target=/opt/pmm-staging \ + install -T -p -m 644 /opt/ansible/ansible.cfg /etc/ansible/ansible.cfg && \ + install -T -p -m 644 /opt/ansible/hosts /etc/ansible/hosts && \ + install -d /usr/share/percona-dashboards && \ + echo "${VERSION}" | install -m 0644 /dev/stdin /usr/share/percona-dashboards/VERSION && \ + ansible-playbook -vvv /opt/ansible/pmm-docker/main.yml && \ + ansible-playbook -vvv /opt/ansible/pmm-docker/post-build.yml && \ + sed -i '/^assumeyes/d' /etc/dnf/dnf.conf + +LABEL org.opencontainers.image.created=${BUILD_DATE} +LABEL org.opencontainers.image.licenses=AGPL-3.0 +LABEL org.opencontainers.image.title="Percona Monitoring and Management" +LABEL org.opencontainers.image.vendor="Percona LLC" +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.percona.pmm=true + +USER pmm + +VOLUME [ "/srv" ] + +EXPOSE 8080 8443 + +HEALTHCHECK --interval=4s --timeout=2s --start-period=10s --retries=3 CMD curl -sf http://127.0.0.1:8080/v1/server/readyz + +CMD ["/opt/entrypoint.sh"] diff --git a/build/pipeline/Dockerfile.server.dockerignore b/build/pipeline/Dockerfile.server.dockerignore new file mode 100644 index 00000000000..5469c95c1e4 --- /dev/null +++ b/build/pipeline/Dockerfile.server.dockerignore @@ -0,0 +1,40 @@ +# Paths are relative to BUILD_ROOT (build/). +# The Dockerfile.server only needs: +# pipeline/output/server/ - built component artifacts +# pipeline/pmm-client.tar.gz - client tarball (bind-mounted in RUN) +# docker/server/ - entrypoint script +# ansible/ - Ansible playbooks + +# Exclude pipeline subdirs that are not part of the image +pipeline/.cache/ +pipeline/scripts/ +pipeline/package/ +pipeline/output/client/ + +# Exclude pipeline metadata/config files +pipeline/Makefile +pipeline/.env +pipeline/.env.example +pipeline/.gitignore +pipeline/AGENT.md +pipeline/README.md +pipeline/SERVER-BUILD.md +pipeline/Dockerfile.builder +pipeline/Dockerfile.client + +# Exclude build-level dirs not needed in the image +scripts/ +packer/ +packages/ +docs/ +Makefile + +# Generic noise +.git/ +*.swp +*.swo +*~ +.DS_Store +*.log +*.rpm +*.deb diff --git a/build/pipeline/Makefile b/build/pipeline/Makefile new file mode 100644 index 00000000000..bb687690aa6 --- /dev/null +++ b/build/pipeline/Makefile @@ -0,0 +1,631 @@ +# Makefile for building PMM components using Docker + +.PHONY: help build + +SHELL := bash +.SHELLFLAGS := -o pipefail -c + +# Determine directory paths regardless of where make is invoked from +PIPELINE_DIR := $(dir $(abspath $(firstword $(MAKEFILE_LIST)))) +BUILD_ROOT := $(abspath $(PIPELINE_DIR)..) + +# Build script location +BUILD_SCRIPT := $(PIPELINE_DIR)/scripts/build-component +PACKAGE_SCRIPT := $(PIPELINE_DIR)/scripts/package-tarball +CLIENT_DOCKER_SCRIPT := $(PIPELINE_DIR)/scripts/build-client-docker +OUTPUT_DIR ?= $(PIPELINE_DIR)/output +PACKAGE_DIR ?= $(PIPELINE_DIR)/package + +# Default target +help: + @echo "PMM Build Targets:" + @echo "" + @echo "Main Targets:" + @echo " make build-client - Build PMM Client components and Docker image" + @echo " make client-tarball - Generate pmm-client tarball from built components" + @echo " make build-server - Build PMM Server Docker image (downloads cache first)" + @echo " make build-all - Build both client and server" + @echo "" + @echo "Component Build Targets:" + @echo " make build COMPONENT= - Build a specific component" + @echo " make build-dynamic COMPONENT= - Build with dynamic linking (GSSAPI)" + @echo " make build-arm64 COMPONENT= - Build for ARM64" + @echo "" + @echo "Docker Image Targets:" + @echo " make build-client-docker - Build PMM Client Docker image" + @echo " make build-server-docker - Build PMM Server Docker image" + @echo " make push-client-docker - Build and push PMM Client Docker image" + @echo "" + @echo "Cache Management:" + @echo " make download-cache - Download repository cache from Minio" + @echo " make update-cache - Fetch upstream changes and upload to Minio" + @echo " make populate-cache - Clone any missing bare repos from upstream (first-time setup)" + @echo " make clean-cache - Remove local cache directory" + @echo " make clean-volumes - Remove Docker cache volumes" + @echo " make clean-all - Remove cache, volumes, and build artifacts" + @echo "" + @echo "Utility Targets:" + @echo " make builder-image - Build the pmm-builder Docker image" + @echo " make clean - Remove output directory" + @echo "" + @echo "Workspace Components:" + @echo " pmm-admin, pmm-agent" + @echo "" + @echo "External Components:" + @echo " node_exporter, mysqld_exporter, mongodb_exporter, postgres_exporter," + @echo " proxysql_exporter, rds_exporter, azure_metrics_exporter, redis_exporter," + @echo " vmagent, nomad, percona-toolkit" + @echo "" + @echo "Environment Variables:" + @echo " PMM_VERSION - Version to build (default: from VERSION file)" + @echo " BUILD_TYPE - Build type: static or dynamic (default: static)" + @echo " GOARCH - Target architecture: amd64 or arm64 (default: amd64)" + @echo " PLATFORM - Docker platform: linux/amd64 or linux/arm64" + @echo " SERVER_PLATFORMS - Server platforms: linux/amd64,linux/arm64 or linux/amd64 (default: linux/amd64)" + @echo " MINIO_ENDPOINT - Minio endpoint (default: http://localhost:9000)" + @echo " MINIO_BUCKET - Minio bucket name (default: cache)" + @echo " OUTPUT_DIR - Output directory (default: ./output)" + @echo "" + @echo "Examples:" + @echo " make build COMPONENT=pmm-admin" + @echo " make build-dynamic COMPONENT=mongodb_exporter" + @echo " make build-arm64 COMPONENT=pmm-agent" + @echo " make server SERVER_PLATFORMS=linux/amd64" + @echo " make download-cache" + @echo " make build-all" + @echo "" + +# Configuration +include .env + +ifeq ($(PMM_VERSION),) +$(error Fatal: PMM_VERSION is not set, exiting...) +endif +BUILD_TYPE ?= static +GOARCH ?= amd64 +PLATFORM ?= linux/amd64 +# Native platform of the build host — containers run natively for speed; +# Go cross-compiles for GOARCH independently. +HOST_ARCH := $(shell uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') +SERVER_PLATFORMS ?= linux/amd64 +GO_VERSION ?= 1.26 +LOG_FILE ?= $(PIPELINE_DIR)/build.log + +# Minio S3 cache configuration +# On macOS/Docker Desktop use the magic hostname; on Linux query the bridge gateway. +_IS_DARWIN := $(filter Darwin,$(shell uname -s)) +_DOCKER_HOST_BRIDGE := $(shell docker network inspect bridge --format='{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null) +_DOCKER_HOST_INTERNAL := $(if $(_IS_DARWIN),host.docker.internal,$(if $(_DOCKER_HOST_BRIDGE),$(_DOCKER_HOST_BRIDGE),$(error Cannot determine Docker bridge gateway. Is Docker running?))) +MINIO_ENDPOINT ?= http://$(_DOCKER_HOST_INTERNAL):9000 +MINIO_BUCKET ?= cache +MINIO_CACHE_PREFIX ?= repos +CACHE_DIR := $(CURDIR)/.cache +REPO_CACHE_DIR := $(CACHE_DIR)/repos + +# Bare repository URLs used by populate-cache and update-cache. +# Keys are the bare directory names stored in REPO_CACHE_DIR; values are the upstream clone URLs. +# Server-side repos: +SERVER_REPO_URLS := \ + pmm.git=https://github.com/percona/pmm.git \ + pmm-dump.git=https://github.com/percona/pmm-dump.git \ + grafana.git=https://github.com/percona/grafana.git \ + VictoriaMetrics.git=https://github.com/VictoriaMetrics/VictoriaMetrics.git +# Client external component repos (vmagent re-uses VictoriaMetrics.git above): +CLIENT_REPO_URLS := \ + node_exporter.git=https://github.com/percona/node_exporter.git \ + mysqld_exporter.git=https://github.com/percona/mysqld_exporter.git \ + mongodb_exporter.git=https://github.com/percona/mongodb_exporter.git \ + postgres_exporter.git=https://github.com/percona/postgres_exporter.git \ + proxysql_exporter.git=https://github.com/percona/proxysql_exporter.git \ + rds_exporter.git=https://github.com/percona/rds_exporter.git \ + azure_metrics_exporter.git=https://github.com/percona/azure_metrics_exporter.git \ + redis_exporter.git=https://github.com/oliver006/redis_exporter.git \ + nomad.git=https://github.com/hashicorp/nomad.git \ + percona-toolkit.git=https://github.com/percona/percona-toolkit.git +ALL_REPO_URLS := $(SERVER_REPO_URLS) $(CLIENT_REPO_URLS) + +# S3 cache flags (reused across all artifact builds) +S3_CACHE_FROM := type=s3,region=us-east-1,bucket=$(MINIO_BUCKET),endpoint_url=$(MINIO_ENDPOINT),access_key_id=$(MINIO_ACCESS_KEY),secret_access_key=$(MINIO_SECRET_KEY),use_path_style=true +S3_CACHE_TO := type=s3,region=us-east-1,bucket=$(MINIO_BUCKET),endpoint_url=$(MINIO_ENDPOINT),access_key_id=$(MINIO_ACCESS_KEY),secret_access_key=$(MINIO_SECRET_KEY),mode=max,use_path_style=true +# Docker configuration +CLIENT_DOCKERFILE ?= $(PIPELINE_DIR)/Dockerfile.client +CLIENT_DOCKER_TAG ?= perconalab/pmm-client:$(PMM_VERSION) +SERVER_DOCKERFILE ?= $(PIPELINE_DIR)/Dockerfile.server +SERVER_DOCKER_TAG ?= perconalab/pmm-server:$(PMM_VERSION) +BUILDX_BUILDER ?= pmm-s3-builder +GOMOD_CACHE_VOL ?= pmm-mod +BUILD_CACHE_VOL ?= pmm-build +YARN_CACHE_VOL ?= pmm-yarn +SERVER_OUTPUT_DIR := $(PIPELINE_DIR)output/server + +export LOG_FILE +export PMM_VERSION +export BUILD_TYPE +export GOARCH +export PLATFORM +export SERVER_PLATFORMS +export GO_VERSION +export OUTPUT_DIR +export PACKAGE_DIR +export MINIO_ENDPOINT +export MINIO_BUCKET +export CACHE_DIR +export REPO_CACHE_DIR +export CLIENT_DOCKERFILE +export CLIENT_DOCKER_TAG +export SERVER_DOCKERFILE +export SERVER_DOCKER_TAG +export PMM_URL +export PMM_REF +export PMM_UI_REF +export PMM_DASHBOARDS_REF +export GRAFANA_URL +export GRAFANA_REF +export VM_URL +export VM_REF +export PMM_DUMP_URL +export PMM_DUMP_REF +export NODE_EXPORTER_URL +export NODE_EXPORTER_REF +export MYSQLD_EXPORTER_URL +export MYSQLD_EXPORTER_REF +export MONGODB_EXPORTER_URL +export MONGODB_EXPORTER_REF +export POSTGRES_EXPORTER_URL +export POSTGRES_EXPORTER_REF +export PROXYSQL_EXPORTER_URL +export PROXYSQL_EXPORTER_REF +export RDS_EXPORTER_URL +export RDS_EXPORTER_REF +export AZURE_METRICS_EXPORTER_URL +export AZURE_METRICS_EXPORTER_REF +export REDIS_EXPORTER_URL +export REDIS_EXPORTER_REF +export VMAGENT_URL +export VMAGENT_REF +export NOMAD_URL +export NOMAD_REF +export PERCONA_TOOLKIT_URL +export PERCONA_TOOLKIT_REF + +# Component lists +WORKSPACE_COMPONENTS := pmm-admin pmm-agent +EXTERNAL_COMPONENTS := node_exporter mysqld_exporter mongodb_exporter postgres_exporter \ + proxysql_exporter rds_exporter azure_metrics_exporter redis_exporter \ + vmagent nomad percona-toolkit +ALL_COMPONENTS := $(WORKSPACE_COMPONENTS) $(EXTERNAL_COMPONENTS) + +# Build the pmm-builder Docker image +builder-image: + @echo "Building pmm-builder image (linux/amd64 + linux/arm64)..." + @docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --build-arg GO_VERSION=$(GO_VERSION) \ + --progress plain \ + --load \ + -t pmm-builder:latest \ + -f Dockerfile.builder \ + . + @echo "pmm-builder image built successfully" + +# Component validation +define check_component + @if [ -z "$(COMPONENT)" ]; then \ + echo "Error: COMPONENT not specified"; \ + echo "Usage: make $(1) COMPONENT="; \ + echo ""; \ + echo "Available components:"; \ + echo " $(ALL_COMPONENTS)"; \ + exit 1; \ + fi +endef + +# Build a single component +build: + $(call check_component,build) + @$(BUILD_SCRIPT) $(COMPONENT) + +# Build with dynamic linking +build-dynamic: + $(call check_component,build-dynamic) + @$(MAKE) build BUILD_TYPE=dynamic COMPONENT=$(COMPONENT) + +# Build for ARM64 +build-arm64: + $(call check_component,build-arm64) + @$(MAKE) build GOARCH=arm64 COMPONENT=$(COMPONENT) + +build-client: +ifeq ($(LOGGING),) + @$(MAKE) LOGGING=1 build-client 2>&1 | tee -a "$(LOG_FILE)" +else + @echo "Building PMM Client..." + @echo "" + @$(MAKE) build-client-components + @echo "" + @$(MAKE) build-client-tarball + @echo "" + @$(MAKE) build-client-docker +endif + +# Build PMM Client components (all client components) +build-client-components: ensure-pmm-builder ensure-build-volumes + @echo "Building PMM Client components..." + @mkdir -p $(OUTPUT_DIR) + @for component in $(ALL_COMPONENTS); do \ + echo ""; \ + echo "================================================="; \ + echo "Building $$component..."; \ + echo "================================================="; \ + $(BUILD_SCRIPT) $$component || exit 1; \ + done + @echo "Artifacts available in: $(OUTPUT_DIR)" + @ls -lh $(OUTPUT_DIR) +# Build PMM Client Docker image +build-client-docker: + @echo "Building PMM Client Docker image..." + @$(CLIENT_DOCKER_SCRIPT) + @echo "" + @echo "Docker image built successfully!" + +# Generate PMM Client tarball package +build-client-tarball: + @echo "Creating pmm-client tarball package..." + @$(PACKAGE_SCRIPT) + +# Build PMM Server (Docker image with all server components) +build-server: +ifeq ($(LOGGING),) + @$(MAKE) LOGGING=1 build-server 2>&1 | tee -a "$(LOG_FILE)" +else + @$(MAKE) download-cache + @$(MAKE) build-server-components + @$(MAKE) generate-version-json + @echo "Assembling PMM Server Docker image..." + @$(MAKE) build-server-docker + @echo "" + @echo "PMM Server built successfully!" +endif + +# Build all components (client + server) +build-all: +ifeq ($(LOGGING),) + @$(MAKE) LOGGING=1 build-all 2>&1 | tee -a "$(LOG_FILE)" +else + @echo "Building all PMM components (client + server)..." + @$(MAKE) build-client + @echo "" + @$(MAKE) build-server + @echo "" + @echo "All components built successfully!" +endif + +# Convenience shortcuts +client: build-client + +server: build-server + +all: build-all + +# Verify mc is installed and MinIO is reachable +check-minio: + @if ! command -v mc >/dev/null 2>&1; then \ + echo "Error: mc (MinIO client) is not installed"; \ + echo "See: https://min.io/docs/minio/linux/reference/minio-mc.html"; \ + exit 1; \ + fi + @echo "Checking MinIO connectivity at $(MINIO_ENDPOINT)..." + @if ! mc ls pmm/$(MINIO_BUCKET) >/dev/null 2>&1; then \ + echo "Error: Cannot reach MinIO at $(MINIO_ENDPOINT)"; \ + echo "Make sure the MinIO container is running and the 'pmm' alias is configured:"; \ + echo " mc alias set pmm $(MINIO_ENDPOINT) "; \ + exit 1; \ + fi + @echo "MinIO is reachable" + +# Download repository cache from Minio (mandatory - fails if unavailable) +download-cache: check-minio + @echo "Downloading repository cache from Minio..." + @echo "Endpoint: $(MINIO_ENDPOINT)" + @echo "Bucket: $(MINIO_BUCKET)/$(MINIO_CACHE_PREFIX)" + @mkdir -p $(REPO_CACHE_DIR) + @echo "Syncing cache from Minio..." + @mc mirror --overwrite pmm/$(MINIO_BUCKET)/$(MINIO_CACHE_PREFIX) $(REPO_CACHE_DIR) || \ + (echo "Error: Failed to download cache from Minio" && exit 1) + @echo "Fixing bare repository structure..." + @for repo in $(REPO_CACHE_DIR)/*.git; do \ + mkdir -p "$$repo/refs/heads" "$$repo/refs/tags" "$$repo/refs/remotes"; \ + done + @echo "Cache download complete" + @echo "Cached repositories:" + @ls -lh $(REPO_CACHE_DIR) + +# Fetch upstream changes into local bare repos and upload the updated cache to Minio +update-cache: check-minio download-cache + @echo "Fetching upstream changes for all cached repositories..." + @for repo in $(REPO_CACHE_DIR)/*.git; do \ + echo "Updating $$repo..."; \ + git -C "$$repo" fetch --all --prune || { echo "Error: failed to fetch $$repo"; exit 1; }; \ + done + @echo "Uploading updated cache to Minio..." + @mc mirror --overwrite $(REPO_CACHE_DIR) pmm/$(MINIO_BUCKET)/$(MINIO_CACHE_PREFIX) || \ + (echo "Error: Failed to upload cache to Minio" && exit 1) + @echo "Cache update complete" + +# Clone any bare repos that are missing from the local cache directory. +# Skips repos that already exist. Useful for first-time setup or when adding new components. +# Run 'make update-cache' afterwards to fetch latest commits and push to Minio. +populate-cache: + @echo "Populating bare-repo cache from upstream (skipping repos that already exist)..." + @mkdir -p $(REPO_CACHE_DIR) + @for item in $(ALL_REPO_URLS); do \ + name=$${item%%=*}; \ + url=$${item##*=}; \ + dest="$(REPO_CACHE_DIR)/$$name"; \ + if [ -d "$$dest" ]; then \ + echo " $$name — already exists, skipping"; \ + else \ + echo "Cloning $$name from $$url ..."; \ + git clone --bare "$$url" "$$dest"; \ + mkdir -p "$$dest/refs/heads" "$$dest/refs/tags" "$$dest/refs/remotes"; \ + fi; \ + done + @echo "Cache populated. Run 'make update-cache' to fetch latest commits and push to Minio." + +# Clean local cache +clean-cache: + @echo "Removing local cache..." + @rm -rf $(CACHE_DIR) + @echo "Local cache removed" + +# Clean build artifacts +clean: + @echo "Cleaning build artifacts..." + @rm -rf $(OUTPUT_DIR) + @echo "Clean complete" + +# Clean Docker volumes +clean-volumes: + @echo "Removing Docker volumes..." + @docker volume rm pmm-mod pmm-build pmm-yarn 2>/dev/null || true + @echo "Volumes removed" + +# Full clean (cache + volumes + output) +clean-all: clean-cache clean-volumes clean + @echo "All caches and build artifacts cleaned" + +# Build PMM Server Docker image (assembly stage only — references pre-built artifact images) +build-server-docker: ensure-builder + @echo "Building PMM Server Docker image..." + @if [ -f "$(PACKAGE_DIR)/pmm-client-$(PMM_VERSION).tar.gz" ]; then \ + cp "$(PACKAGE_DIR)/pmm-client-$(PMM_VERSION).tar.gz" pmm-client.tar.gz; \ + else \ + echo "Error: pmm-client tarball not found at $(PACKAGE_DIR)/pmm-client-$(PMM_VERSION).tar.gz"; \ + echo "Run 'make build-client-tarball' first to create the tarball"; \ + exit 1; \ + fi + @docker buildx build \ + --load \ + --builder $(BUILDX_BUILDER) \ + --platform $(SERVER_PLATFORMS) \ + --progress plain \ + --cache-from $(S3_CACHE_FROM) --cache-to $(S3_CACHE_TO) \ + --build-arg BUILD_DATE="$$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --build-arg VERSION="$(PMM_VERSION)" \ + -f $(SERVER_DOCKERFILE) \ + -t $(SERVER_DOCKER_TAG) \ + $(BUILD_ROOT) + @rm -f pmm-client.tar.gz + @docker buildx stop $(BUILDX_BUILDER) 2>/dev/null || true + @echo "Docker image built successfully!" + +# Build and push PMM Client Docker image +push-client-docker: + @echo "Building and pushing PMM Client Docker image..." + @PUSH_DOCKER=1 $(CLIENT_DOCKER_SCRIPT) + @echo "Docker image pushed successfully!" + +# Generate VERSION.json with all component git refs. +generate-version-json: + @echo "Generating VERSION.json..." + @$(PIPELINE_DIR)/scripts/generate-version-json + +# Ensure a buildx builder with the docker-container driver exists and is running. +# The default "docker" driver does not support remote cache backends (s3, registry, etc.). +ensure-builder: + @docker buildx inspect $(BUILDX_BUILDER) >/dev/null 2>&1 || \ + docker buildx create --name $(BUILDX_BUILDER) --driver docker-container + @docker buildx inspect $(BUILDX_BUILDER) --bootstrap >/dev/null + +# Ensure pmm-builder image exists (used for docker run-based component builds) +ensure-pmm-builder: + @docker image inspect pmm-builder:latest >/dev/null 2>&1 || $(MAKE) builder-image + +# Ensure Docker volumes exist for Go module, build, and Yarn caches +ensure-build-volumes: + @docker volume inspect $(GOMOD_CACHE_VOL) >/dev/null 2>&1 || docker volume create $(GOMOD_CACHE_VOL) + @docker volume inspect $(BUILD_CACHE_VOL) >/dev/null 2>&1 || docker volume create $(BUILD_CACHE_VOL) + @docker volume inspect $(YARN_CACHE_VOL) >/dev/null 2>&1 || docker volume create $(YARN_CACHE_VOL) + +# --------------------------------------------------------------------------- +# Individual server component builds (each produces a local Docker image) +# --------------------------------------------------------------------------- + +build-pmm-managed: ensure-pmm-builder ensure-build-volumes + @echo "Building pmm-managed binaries..." + @mkdir -p \ + $(SERVER_OUTPUT_DIR)/pmm-managed/bin \ + $(SERVER_OUTPUT_DIR)/pmm-managed/data/swagger \ + $(SERVER_OUTPUT_DIR)/pmm-managed/data/advisors \ + $(SERVER_OUTPUT_DIR)/pmm-managed/data/checks \ + $(SERVER_OUTPUT_DIR)/pmm-managed/data/alerting-templates + @docker run --rm \ + --platform linux/$(HOST_ARCH) \ + -v $(REPO_CACHE_DIR)/pmm.git:/repos/pmm.git:ro \ + -v $(SERVER_OUTPUT_DIR)/pmm-managed:/output \ + -v $(GOMOD_CACHE_VOL):/go/pkg/mod \ + -v $(BUILD_CACHE_VOL):/root/.cache/go-build \ + -e GOCACHE=/root/.cache/go-build \ + -e GOARCH=$(GOARCH) \ + pmm-builder:latest bash -c "\ + set -e; git clone /repos/pmm.git /build/pmm; \ + cd /build/pmm; git checkout $(PMM_REF); \ + make -C managed release GOARCH=$(GOARCH); \ + cd /build/pmm/qan-api2 && make release GOARCH=$(GOARCH); \ + cd /build/pmm/vmproxy && make release GOARCH=$(GOARCH); \ + cp /build/pmm/bin/pmm-managed /build/pmm/bin/pmm-encryption-rotation \ + /build/pmm/bin/pmm-managed-init /build/pmm/bin/pmm-managed-starlark \ + /build/pmm/bin/qan-api2 /build/pmm/bin/vmproxy /output/bin/; \ + cp -r /build/pmm/api/swagger/. /output/data/swagger/; \ + cp /build/pmm/managed/data/advisors/*.yml /output/data/advisors/; \ + cp /build/pmm/managed/data/checks/*.yml /output/data/checks/; \ + cp /build/pmm/managed/data/alerting-templates/*.yml /output/data/alerting-templates/" + +build-pmm-dump: ensure-pmm-builder ensure-build-volumes + @echo "Building pmm-dump binary..." + @mkdir -p $(SERVER_OUTPUT_DIR)/pmm-dump + @docker run --rm \ + --platform linux/$(HOST_ARCH) \ + -v $(REPO_CACHE_DIR)/pmm-dump.git:/repos/pmm-dump.git:ro \ + -v $(SERVER_OUTPUT_DIR)/pmm-dump:/output \ + -v $(GOMOD_CACHE_VOL):/go/pkg/mod \ + -v $(BUILD_CACHE_VOL):/root/.cache/go-build \ + -e GOCACHE=/root/.cache/go-build \ + -e GOARCH=$(GOARCH) \ + pmm-builder:latest bash -c "\ + set -e; git clone /repos/pmm-dump.git /build/pmm-dump; \ + cd /build/pmm-dump; git checkout $(PMM_DUMP_REF); \ + CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) make build; \ + cp pmm-dump /output/" + +build-grafana-go: ensure-build-volumes + @echo "Building Grafana Go binaries..." + @mkdir -p $(SERVER_OUTPUT_DIR)/grafana-go + @docker run --rm \ + --platform linux/$(GOARCH) \ + -v $(REPO_CACHE_DIR)/grafana.git:/repos/grafana.git:ro \ + -v $(SERVER_OUTPUT_DIR)/grafana-go:/output \ + -v $(GOMOD_CACHE_VOL):/go/pkg/mod \ + -v $(BUILD_CACHE_VOL):/root/.cache/go-build \ + -e GOCACHE=/root/.cache/go-build \ + -e GOARCH=$(GOARCH) \ + golang:$(GO_VERSION) bash -c "\ + set -e; git clone /repos/grafana.git /build/grafana; \ + cd /build/grafana; git checkout $(GRAFANA_REF); \ + sed -i 's/unknown-dev/11.6.4/' pkg/build/git.go; \ + make build-go; \ + cp bin/linux-$(GOARCH)/grafana-server bin/linux-$(GOARCH)/grafana bin/linux-$(GOARCH)/grafana-cli /output/" + +build-grafana-ui: ensure-build-volumes + @echo "Building Grafana UI assets..." + @mkdir -p $(SERVER_OUTPUT_DIR)/grafana-ui + @docker run --rm \ + --platform linux/$(HOST_ARCH) \ + -v $(REPO_CACHE_DIR)/grafana.git:/repos/grafana.git:ro \ + -v $(SERVER_OUTPUT_DIR)/grafana-ui:/output \ + -v $(YARN_CACHE_VOL):/yarn-cache \ + -e YARN_CACHE_FOLDER=/yarn-cache \ + -e YARN_GLOBAL_FOLDER=/yarn-cache \ + -e NODE_OPTIONS="--max-old-space-size=4096" \ + node:22 bash -c "\ + set -e; npm install -g grunt-cli; \ + git clone /repos/grafana.git /build/grafana; \ + cd /build/grafana; git checkout $(GRAFANA_REF); \ + make deps-js && make build-js; \ + cp -r public conf tools /output/" + +build-victoriametrics: ensure-pmm-builder ensure-build-volumes + @echo "Building VictoriaMetrics binaries..." + @mkdir -p $(SERVER_OUTPUT_DIR)/victoriametrics + @docker run --rm \ + --platform linux/$(HOST_ARCH) \ + -v $(REPO_CACHE_DIR)/VictoriaMetrics.git:/repos/VictoriaMetrics.git:ro \ + -v $(SERVER_OUTPUT_DIR)/victoriametrics:/output \ + -v $(GOMOD_CACHE_VOL):/go/pkg/mod \ + -v $(BUILD_CACHE_VOL):/root/.cache/go-build \ + -e GOCACHE=/root/.cache/go-build \ + -e GOARCH=$(GOARCH) \ + pmm-builder:latest bash -c "\ + set -e; git clone /repos/VictoriaMetrics.git /build/VictoriaMetrics; \ + cd /build/VictoriaMetrics; git checkout $(VM_REF); \ + make victoria-metrics-pure vmalert-pure PKG_TAG=$(VM_REF) BUILDINFO_TAG=$(VM_REF); \ + cp bin/victoria-metrics-pure bin/vmalert-pure /output/" + +build-pmm-dashboards: ensure-build-volumes + @echo "Building PMM dashboards assets..." + @mkdir -p $(SERVER_OUTPUT_DIR)/pmm-dashboards + @docker run --rm \ + --platform linux/$(HOST_ARCH) \ + -v $(REPO_CACHE_DIR)/pmm.git:/repos/pmm.git:ro \ + -v $(SERVER_OUTPUT_DIR)/pmm-dashboards:/output \ + -v $(YARN_CACHE_VOL):/yarn-cache \ + -e YARN_CACHE_FOLDER=/yarn-cache \ + -e YARN_GLOBAL_FOLDER=/yarn-cache \ + -e NODE_OPTIONS="--max-old-space-size=4096" \ + node:22 bash -c "\ + set -e; git clone /repos/pmm.git /build/pmm; \ + cd /build/pmm; git checkout $(PMM_DASHBOARDS_REF); \ + cd dashboards && make release; \ + cp -r panels /output/panels; \ + cp -r pmm-app/dist /output/pmm-app-dist" + +build-pmm-ui: ensure-build-volumes + @echo "Building PMM UI assets..." + @mkdir -p $(SERVER_OUTPUT_DIR)/pmm-ui + @docker run --rm \ + --platform linux/$(HOST_ARCH) \ + -v $(REPO_CACHE_DIR)/pmm.git:/repos/pmm.git:ro \ + -v $(SERVER_OUTPUT_DIR)/pmm-ui:/output \ + -v $(YARN_CACHE_VOL):/yarn-cache \ + -e YARN_CACHE_FOLDER=/yarn-cache \ + -e YARN_GLOBAL_FOLDER=/yarn-cache \ + -e NODE_OPTIONS="--max-old-space-size=4096" \ + node:22 bash -c "\ + set -e; git clone /repos/pmm.git /build/pmm-ui; \ + cd /build/pmm-ui; git checkout $(PMM_UI_REF); \ + cd ui && make release; \ + cp -r /build/pmm-ui/ui/apps/pmm/dist /output/pmm-dist; \ + cp -r /build/pmm-ui/ui/apps/pmm-compat/dist /output/pmm-compat-dist" + +# --------------------------------------------------------------------------- +# Orchestration: Go builds in parallel, then Node builds sequentially +# --------------------------------------------------------------------------- + +build-server-components: + @echo "Building server component images (Go stages in parallel)..." + @$(MAKE) -j4 build-pmm-managed build-pmm-dump \ + build-grafana-go build-victoriametrics + @echo "" + @echo "Building server component images (Node stages sequentially)..." + @$(PIPELINE_DIR)/scripts/check-ui-cache \ + "grafana" "$(GRAFANA_REF)" \ + "$(REPO_CACHE_DIR)/grafana.git" \ + "$(SERVER_OUTPUT_DIR)/grafana-ui" \ + "$(SERVER_OUTPUT_DIR)/version.json" \ + && $(MAKE) build-grafana-ui || true + @$(PIPELINE_DIR)/scripts/check-ui-cache \ + "pmm-dashboards" "$(PMM_DASHBOARDS_REF)" \ + "$(REPO_CACHE_DIR)/pmm.git" \ + "$(SERVER_OUTPUT_DIR)/pmm-dashboards" \ + "$(SERVER_OUTPUT_DIR)/version.json" \ + && $(MAKE) build-pmm-dashboards || true + @$(PIPELINE_DIR)/scripts/check-ui-cache \ + "pmm" "$(PMM_UI_REF)" \ + "$(REPO_CACHE_DIR)/pmm.git" \ + "$(SERVER_OUTPUT_DIR)/pmm-ui" \ + "$(SERVER_OUTPUT_DIR)/version.json" \ + && $(MAKE) build-pmm-ui || true + @echo "All server component images built successfully" + +# Remove intermediate server artifact images +clean-server-artifacts: + @echo "Removing server artifact images..." + @docker rmi \ + pmm-build/pmm-dashboards:$(PMM_BUILD_TAG) \ + pmm-build/pmm-ui:$(PMM_BUILD_TAG) \ + 2>/dev/null || true + @rm -rf $(SERVER_OUTPUT_DIR)/{victoriametrics,pmm-managed,pmm-dump,grafana-go,grafana-ui} + @echo "Server artifact images removed" + +check: + @echo "PIPELINE_DIR: $(PIPELINE_DIR)" diff --git a/build/pipeline/README.md b/build/pipeline/README.md new file mode 100644 index 00000000000..0cf72e1f82a --- /dev/null +++ b/build/pipeline/README.md @@ -0,0 +1,504 @@ +# PMM Build Pipeline + +Docker-based build system for PMM components. Server components are each built with `docker run`, outputting artifacts directly to the host via Docker volumes. The final server image is assembled in a single-stage `FROM oraclelinux:9-slim` Dockerfile using BuildKit. + +## Setup + +Before building, copy the example environment file and customize if needed: + +```bash +cd build/pipeline +cp .env.example .env +``` + +The `.env` file contains git refs for all external dependencies (Grafana, VictoriaMetrics, exporters, etc.). You can modify these to build with different versions. + +### Minio Requirement for Server Builds + +PMM Server and Client builds require a local Minio instance for repository cache. See [Cache Management](#cache-management) section for setup instructions. + +Quick Minio setup: + +```bash +# Start Minio container +docker run -d --name minio -p 9000:9000 -p 9001:9001 \ + -e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data --console-address ":9001" + +# Install and configure mc (Minio client) +brew install minio/stable/mc # macOS +mc alias set pmm http://localhost:9000 minioadmin minioadmin +mc mb pmm/cache + +# Populate cache (see Cache Maintenance section for details) +``` + +## Quick Start + +```bash +cd build/pipeline + +# Build all PMM Client components (binaries only) +make build-client + +# Build PMM Client Docker image +make build-client-docker + +# Build PMM Client tarball +make build-client-tarball + +# Build everything for client (components + Docker image + tarball) +make client + +# Build PMM Server (Docker image) - defaults to linux/amd64 +make server + +# Build PMM Server for ARM64 +make server SERVER_PLATFORMS=linux/arm64 + +# Build everything (client + server) +make all + +# Build a single component +make build COMPONENT=pmm-admin + +# Build with dynamic linking (GSSAPI support) +make build-dynamic COMPONENT=mongodb_exporter + +# Build for ARM64 +make build-arm64 COMPONENT=pmm-agent + +# Build the pmm-builder Docker image (optional - auto-built on first use) +make builder-image +``` + +## Components + +### Workspace Components +Built from the PMM repository workspace: +- **pmm-admin** - PMM client CLI tool +- **pmm-agent** - PMM agent daemon + +### External Components +Built from external Git repositories: +- **node_exporter** - Prometheus Node Exporter (Percona fork) +- **mysqld_exporter** - MySQL Server Exporter (Percona fork) +- **mongodb_exporter** - MongoDB Exporter (Percona fork) +- **postgres_exporter** - PostgreSQL Exporter (Percona fork) +- **proxysql_exporter** - ProxySQL Exporter (Percona fork) +- **rds_exporter** - AWS RDS Exporter (Percona fork) +- **azure_metrics_exporter** - Azure Metrics Exporter (Percona fork) +- **redis_exporter** - Redis Exporter +- **vmagent** - VictoriaMetrics Agent +- **nomad** - HashiCorp Nomad +- **percona-toolkit** - Percona Toolkit utilities + +## Environment Variables + +### Build Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `PMM_VERSION` | Version to build | From `VERSION` file | +| `BUILD_TYPE` | Build type: `static` or `dynamic` | `static` | +| `GOARCH` | Target architecture: `amd64` or `arm64` | `amd64` | +| `PLATFORM` | Docker platform: `linux/amd64` or `linux/arm64` | `linux/amd64` | +| `SERVER_PLATFORMS` | Server build platforms (comma-separated) | `linux/amd64` | +| `GO_VERSION` | Go version for builder image | `1.26` | +| `HOST_ARCH` | Native arch of build host (pure-Go/Node containers run natively) | auto-detected | +| `GOMOD_CACHE_VOL` | Docker volume for Go module cache | `pmm-mod` | +| `BUILD_CACHE_VOL` | Docker volume for Go build cache | `pmm-build` | +| `YARN_CACHE_VOL` | Docker volume for Yarn package cache | `pmm-yarn` | +| `OUTPUT_DIR` | Output directory for artifacts | `./output` | +| `PACKAGE_DIR` | Output directory for tarball packages | `./package` | +| `MINIO_ENDPOINT` | Minio S3 endpoint URL | `http://localhost:9000` | +| `MINIO_BUCKET` | Minio bucket name | `cache` | +| `MINIO_CACHE_PREFIX` | Cache prefix in bucket | `repos` | + +### Component Versions (.env file) + +Component versions are managed via the `.env` file. Copy `.env.example` to `.env` and customize as needed: + +**Server Components:** +- `PMM_REF` - PMM (pmm-managed, qan-api2, vmproxy, UI) git ref (branch/tag/commit) +- `GRAFANA_REF` - Grafana git ref (branch/tag/commit) +- `VM_REF` - VictoriaMetrics git ref (branch/tag/commit) +- `DASHBOARDS_REF` - percona-dashboards git ref (branch/tag/commit) +- `PMM_DUMP_REF` - pmm-dump git ref (branch/tag/commit) + +**Client Components:** +- `REDIS_EXPORTER_REF` - redis_exporter git ref (branch/tag/commit) +- `VMAGENT_REF` - vmagent git ref (branch/tag/commit) +- `NOMAD_REF` - nomad git ref (branch/tag/commit) + + +## Build Process + +The build system: + +1. **Builds the pmm-builder Docker image** (if not already present) based on `golang:latest` +2. **Creates Docker volumes** for caching Go modules, build cache, and Yarn packages +3. **Fetches component metadata** from `.gitmodules` in pmm-submodules repository +4. **Runs server component builds** using `docker run` — each component writes artifacts to + `output/server//` on the host via a volume mount: + - **pmm-managed, pmm-dump, VictoriaMetrics**: `pmm-builder:latest` (pure Go, CGO disabled) + - **grafana-go**: `golang:$(GO_VERSION)` (CGO enabled for SQLite; runs at target `GOARCH`) + - **grafana-ui, pmm-dashboards, pmm-ui**: `node:22` with shared Yarn cache +5. **Assembles the server runtime image** from the host-side artifacts using + `docker buildx build` (BuildKit with S3 layer cache) +6. **Outputs artifacts** to the configured output directory + +### Workspace Components + +Workspace components (pmm-admin, pmm-agent) are built directly from the PMM repository using their respective Makefiles: + +```bash +make build COMPONENT=pmm-admin +``` + +### External Components + +External components are cloned from their Git repositories and built: + +```bash +make build COMPONENT=mysqld_exporter +``` + +## Cache Management + +PMM uses **Minio S3** for persistent build cache storage across ephemeral build agents. + +### Cache Strategy + +- **One-way sync**: Cache is downloaded from Minio to local disk before builds +- **No upload**: Local cache is never synced back to avoid conflicts between parallel builds +- **Mandatory**: Both server and client builds fail hard if the bare repo cache is unavailable — no internet fallback +- **Cache maintenance**: A separate process/job maintains the Minio cache (see Cache Maintenance below) + +### Local Minio Setup + +Start a local Minio container for development: + +```bash +docker run -d \ + --name minio \ + -p 9000:9000 \ + -p 9001:9001 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + -v minio-data:/data \ + minio/minio server /data --console-address ":9001" +``` + +Configure Minio client (mc): + +```bash +# Install mc +brew install minio/stable/mc # macOS +# Or for Linux: https://min.io/docs/minio/linux/reference/minio-mc.html + +# Configure alias +mc alias set pmm http://127.0.0.1:9000 minioadmin minioadmin + +# Create bucket +mc mb pmm/cache +``` + +### Minio Configuration + +Configure Minio access via environment variables or `.env` file: + +```bash +MINIO_ENDPOINT=http://127.0.0.1:9000 +MINIO_BUCKET=cache +MINIO_CACHE_PREFIX=repos +``` + +### Cache Targets + +```bash +# Clone any missing bare repos directly from upstream (first-time / new component setup) +make populate-cache + +# Fetch latest upstream commits and push everything to Minio +make update-cache + +# Download repository cache from Minio (mandatory for server and client builds) +make download-cache + +# Build server (downloads cache first, fails if Minio unavailable) +make server + +# Clean local cache only +make clean-cache + +# Clean Docker volumes only +make clean-volumes + +# Clean everything (cache + volumes + output) +make clean-all +``` + +### Cache Structure + +The `.cache/repos/` directory contains bare Git repositories for all components: + +``` +.cache/ +└── repos/ + ├── azure_metrics_exporter.git/ # client + ├── grafana.git/ # server + ├── mongodb_exporter.git/ # client + ├── mysqld_exporter.git/ # client + ├── node_exporter.git/ # client + ├── nomad.git/ # client + ├── percona-toolkit.git/ # client + ├── pmm-dump.git/ # server + ├── pmm.git/ # server + UI + ├── postgres_exporter.git/ # client + ├── proxysql_exporter.git/ # client + ├── rds_exporter.git/ # client + ├── redis_exporter.git/ # client + └── VictoriaMetrics.git/ # server + vmagent +``` + +Server builds mount these repos read-only into each `docker run` build container. +Client builds (`build-component`) also require them — a missing bare repo is a hard failure, +consistent with server build behaviour. + +**Note:** The `download-cache` Make target automatically fixes bare repository structure by creating empty `refs` subdirectories. This is necessary because some sync tools (like `mc mirror`) don't preserve empty directories. + +### Cache Maintenance (for build administrators) + +To populate or update the Minio cache (run from a dedicated maintenance job, not build agents): + +> **Shortcut:** `make populate-cache` clones any missing repos automatically, then `make update-cache` fetches the latest commits and pushes everything to Minio. + +```bash +# Create cache directory +mkdir -p /tmp/pmm-cache && cd /tmp/pmm-cache + +# Clone repositories as bare for efficiency +# Server components: +git clone --bare https://github.com/percona/pmm.git pmm.git +git clone --bare https://github.com/percona/pmm-dump.git pmm-dump.git +git clone --bare https://github.com/percona/grafana.git grafana.git +git clone --bare https://github.com/VictoriaMetrics/VictoriaMetrics.git VictoriaMetrics.git +# Client components (vmagent uses VictoriaMetrics.git above): +git clone --bare https://github.com/percona/node_exporter.git node_exporter.git +git clone --bare https://github.com/percona/mysqld_exporter.git mysqld_exporter.git +git clone --bare https://github.com/percona/mongodb_exporter.git mongodb_exporter.git +git clone --bare https://github.com/percona/postgres_exporter.git postgres_exporter.git +git clone --bare https://github.com/percona/proxysql_exporter.git proxysql_exporter.git +git clone --bare https://github.com/percona/rds_exporter.git rds_exporter.git +git clone --bare https://github.com/percona/azure_metrics_exporter.git azure_metrics_exporter.git +git clone --bare https://github.com/oliver006/redis_exporter.git redis_exporter.git +git clone --bare https://github.com/hashicorp/nomad.git nomad.git +git clone --bare https://github.com/percona/percona-toolkit.git percona-toolkit.git + +# Fix bare repository structure (git requires refs directories to exist) +for repo in *.git; do + mkdir -p "$repo/refs/heads" "$repo/refs/tags" "$repo/refs/remotes" +done + +# Update existing bare repositories (if refreshing cache) +for repo in *.git; do + cd "$repo" && git fetch --all --prune && cd .. +done + +# Upload to Minio +mc mirror --overwrite . pmm/cache/repos/ +``` + +**Recommended**: Set up a scheduled job (e.g., daily cron) to keep the cache fresh. + +### Troubleshooting + +**Error: mc is not installed** +```bash +brew install minio/stable/mc # macOS +# Or see: https://min.io/docs/minio/linux/reference/minio-mc.html +``` + +**Error: Failed to download cache from Minio** +- Ensure Minio container is running: `docker ps | grep minio` +- Check Minio endpoint: `mc ls pmm/cache/` +- Verify bucket exists: `mc mb pmm/cache` +- Populate cache: See Cache Maintenance section above + +## Directory Structure + +``` +build/ +├── pipeline/ +│ ├── Makefile # Main build targets +│ ├── README.md # This file +│ ├── Dockerfile.client # PMM Client Docker image +│ ├── Dockerfile.server # PMM Server assembly (copies host-built artifacts) +│ ├── Dockerfile.builder # pmm-builder image for Go component builds +│ ├── output/ +│ │ └── server/ # Per-component artifact directories (created by builds) +│ │ ├── pmm-managed/ # 6 Go binaries + swagger + YAML data dirs +│ │ ├── pmm-dump/ # pmm-dump binary +│ │ ├── grafana-go/ # grafana-server, grafana, grafana-cli +│ │ ├── grafana-ui/ # public/, conf/, tools/ +│ │ ├── victoriametrics/ # victoria-metrics-pure, vmalert-pure +│ │ ├── pmm-dashboards/ # panels/, pmm-app-dist/ +│ │ └── pmm-ui/ # pmm-dist/, pmm-compat-dist/ +│ └── package/ # Tarballs (created) +└── scripts/ + ├── build-component # Component build script + ├── package-tarball # Tarball packaging script + └── build-client-docker # Client Docker build script +``` + +## Build Targets + +### Main Targets + +- **`make client`** - Builds all client components, Docker image, and tarball +- **`make server`** - Builds the PMM Server Docker image +- **`make all`** - Builds both client and server + +### Client Build Targets + +- **`make build-client`** - Build all client components (binaries only) +- **`make build-client-docker`** - Build PMM Client Docker image +- **`make build-client-tarball`** - Build PMM Client tarball package + +### Server Build Targets + +- **`make build-server`** - Build PMM Server Docker image (multi-architecture) +- **`make build-server-docker`** - Same as build-server + +### Component Targets + +- **`make build COMPONENT=`** - Build a specific component +- **`make build-dynamic COMPONENT=`** - Build with dynamic linking (GSSAPI) +- **`make build-arm64 COMPONENT=`** - Build for ARM64 + +### Utility Targets + +- **`make builder-image`** - Build the pmm-builder Docker image +- **`make gitmodules`** - Build gitmodules parser binary +- **`make clean`** - Remove output directory +- **`make clean-volumes`** - Remove Docker cache volumes + +## Examples + +### Build PMM Client + +```bash +cd build/pipeline +make client +``` + +This will: +1. Build all client components (pmm-admin, pmm-agent, exporters) +2. Create the PMM Client Docker image +3. Generate the PMM Client tarball package +4. Output artifacts to `./output/` and `./package/` + +### Build PMM Server + +```bash +cd build/pipeline +make server +``` + +This builds the PMM Server Docker image via a two-phase process: + +**Phase 1 — component builds** (all `docker run`, no BuildKit): +1. Go components built in parallel (`-j4`): pmm-managed/qan-api2/vmproxy, pmm-dump, Grafana backend, VictoriaMetrics +2. Node.js assets built sequentially (to avoid Yarn network saturation): Grafana UI, PMM UI, percona-dashboards + +Each component writes its artifacts to `output/server//` on the host. Pure-Go and Node containers run natively at `HOST_ARCH` (auto-detected from `uname -m`); Go cross-compiles for `GOARCH` independently. grafana-go (needs CGO/SQLite) runs at `--platform linux/$(GOARCH)` instead. + +**Phase 2 — image assembly** (`docker buildx build` with BuildKit S3 layer cache): +- Copies host-side artifacts into a single `FROM oraclelinux:9-slim` image + +**Default Architecture:** `linux/amd64` + +Set `GOARCH=arm64` to cross-compile binaries for ARM64: + +```bash +# Build for ARM64 +make server GOARCH=arm64 SERVER_PLATFORMS=linux/arm64 +``` + +### Build Everything + +```bash +cd build/pipeline +make all +``` + +### Build pmm-admin + +```bash +cd build/pipeline +make build COMPONENT=pmm-admin +``` + +### Custom build with specific version + +```bash +PMM_VERSION=3.0.0-rc1 make build COMPONENT=pmm-agent +``` + +### Build for ARM64 with dynamic linking + +```bash +BUILD_TYPE=dynamic GOARCH=arm64 make build COMPONENT=mongodb_exporter +``` + +### Create distribution tarball + +First build all components, then package them: + +```bash +make build-client +make build-client-tarball +``` + +The tarball will be created at `./package/pmm-client-${VERSION}.tar.gz` with the following structure: + +``` +pmm-client-${VERSION}/ +├── bin/ # All built binaries (pmm-admin, pmm-agent, exporters, etc.) +├── config/ # Configuration files (systemd services) +├── debian/ # Debian packaging files +├── rpm/ # RPM spec files +├── queries-*.yml # Query examples (if present) +├── install_tarball # Installation script +└── VERSION # Version identifier +``` + +## Troubleshooting + +### Build fails with "No URL/ref found" + +The component might not be in `.gitmodules`. Check the fallback values in `build-component.sh`. + +### Permission denied errors + +Ensure Docker is running and you have permissions to create volumes. + +### Build artifacts not found + +Check `OUTPUT_DIR` (default: `./output`). Verify the build completed successfully. + +## CI/CD Integration + +```yaml +- name: Build PMM Components + run: | + cd build/pipeline + make volumes + make build-all + env: + PMM_VERSION: ${{ github.ref_name }} +``` + +## License +This build pipeline is licensed under the [Apache License 2.0](LICENSE). diff --git a/build/pipeline/scripts/build-client-docker b/build/pipeline/scripts/build-client-docker new file mode 100755 index 00000000000..25338205d69 --- /dev/null +++ b/build/pipeline/scripts/build-client-docker @@ -0,0 +1,94 @@ +#!/bin/bash +# Build PMM Client Docker image + +set -o errexit +set -o pipefail +set -o nounset + +# Default values +SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +PIPELINE_DIR="$(dirname "${SCRIPT_DIR}")" +PMM_VERSION="${PMM_VERSION:?No PMM_VERSION specified}" +PACKAGE_DIR="${PACKAGE_DIR:-${PIPELINE_DIR}/package}" +DOCKER_BUILD_DIR="${PIPELINE_DIR}" + +# Docker configuration +DOCKERFILE="${DOCKERFILE:-Dockerfile.client}" +DOCKER_TAG="${DOCKER_TAG:-perconalab/pmm-client:${PMM_VERSION}}" +PLATFORM="${PLATFORM:-linux/amd64}" + +# Package name +PACKAGE_NAME="pmm-client-${PMM_VERSION}" +TARBALL="${PACKAGE_DIR}/${PACKAGE_NAME}.tar.gz" + +usage() { + cat < + +Build a PMM component using Docker. + +Components: + Workspace components: + pmm-admin, pmm-agent + + External components: + node_exporter, mysqld_exporter, mongodb_exporter, postgres_exporter, + proxysql_exporter, rds_exporter, azure_metrics_exporter, redis_exporter, + vmagent, nomad, percona-toolkit + +Environment Variables: + PMM_VERSION - Version to build (default: from VERSION file) + BUILD_TYPE - Build type: static or dynamic (default: static) + GOARCH - Target architecture: amd64 or arm64 (default: amd64) + PLATFORM - Docker platform: linux/amd64 or linux/arm64 (default: linux/amd64) + OUTPUT_DIR - Output directory for artifacts (default: ./output) + +Examples: + $0 pmm-admin + PMM_VERSION=3.0.0 BUILD_TYPE=dynamic $0 mongodb_exporter +EOF + exit 1 +} + +build_builder_image() { + if ! docker image inspect "${BUILDER_IMAGE}" >/dev/null 2>&1; then + echo "Building pmm-builder image..." + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --build-arg GO_VERSION="${GO_VERSION}" \ + --progress=plain \ + --load \ + -t "${BUILDER_IMAGE}" \ + -f "${BUILDER_DOCKERFILE}" \ + "$(dirname "${BUILDER_DOCKERFILE}")" + fi +} + +create_volumes() { + echo "Creating Docker volumes for caching..." + if ! docker volume inspect "${GOMOD_CACHE_VOL}" >/dev/null 2>&1; then + docker volume create "${GOMOD_CACHE_VOL}" + fi + if ! docker volume inspect "${BUILD_CACHE_VOL}" >/dev/null 2>&1; then + docker volume create "${BUILD_CACHE_VOL}" + fi +} + +build_workspace_component() { + local component=$1 + local subdir + + # Determine subdirectory based on component name + case "${component}" in + pmm-admin) subdir="admin" ;; + pmm-agent) subdir="agent" ;; + *) + echo "Error: Unknown workspace component '${component}'" >&2 + return 1 + ;; + esac + + echo "Building workspace component: ${component} (${BUILD_TYPE}, ${GOARCH})" + + mkdir -p "${OUTPUT_DIR}" + + # Build command and CGO setting depend on component. + # pmm-agent uses CGO_ENABLED=1 with -static linking, so the container must + # run on the target architecture (GOARCH) to get a matching gcc toolchain. + # pmm-admin is CGO_ENABLED=0 and can run on the native host arch. + local build_cmd + local cgo_enabled + local container_arch + case "${component}" in + pmm-admin) + build_cmd="make -C /workspace/${subdir} release PMM_RELEASE_PATH=/output" + cgo_enabled=0 + container_arch="${HOST_ARCH}" + ;; + pmm-agent) + if [ "${BUILD_TYPE}" = "dynamic" ]; then + build_cmd="make -C /workspace/${subdir} release-gssapi PMM_RELEASE_PATH=/output" + else + build_cmd="make -C /workspace/${subdir} release PMM_RELEASE_PATH=/output" + fi + cgo_enabled=1 + container_arch="${GOARCH}" + ;; + esac + + docker run --rm \ + --platform "linux/${container_arch}" \ + -v "${WORKSPACE_DIR}:/workspace:ro" \ + -v "${OUTPUT_DIR}:/output" \ + -v "${GOMOD_CACHE_VOL}:/go/pkg/mod" \ + -v "${BUILD_CACHE_VOL}:/root/.cache/go-build" \ + -e GOCACHE=/root/.cache/go-build \ + -e CGO_ENABLED="${cgo_enabled}" \ + -e GOOS=linux \ + -e GOARCH="${GOARCH}" \ + -e PMM_RELEASE_VERSION="${PMM_VERSION}" \ + -e PMM_RELEASE_TIMESTAMP="$(date '+%s')" \ + -e PMM_RELEASE_PATH=/output \ + -e PMM_RELEASE_FULLCOMMIT="$(git -C "${WORKSPACE_DIR}" rev-parse HEAD 2>/dev/null || echo "unknown")" \ + -e PMM_RELEASE_BRANCH="$(git -C "${WORKSPACE_DIR}" describe --always --contains --all 2>/dev/null || echo "unknown")" \ + -w /workspace \ + "${BUILDER_IMAGE}" \ + bash -c "${build_cmd}" + + echo "Build complete! Artifacts saved to ${OUTPUT_DIR}" + ls -lh "${OUTPUT_DIR}" +} + +build_external_component() { + local component=$1 + local url + local ref + + # Resolve URL and ref from .env variables (e.g. node_exporter → NODE_EXPORTER_URL / _REF). + local env_prefix + env_prefix="$(echo "${component}" | tr '[:lower:]-' '[:upper:]_')" + local env_url="${env_prefix}_URL" + local env_ref="${env_prefix}_REF" + url="${!env_url:-}" + ref="${!env_ref:-}" + + if [ -z "${url}" ] || [ -z "${ref}" ]; then + echo "Error: ${env_url} or ${env_ref} is not set in .env" >&2 + echo "Run 'scripts/migrate-from-submodules' to populate missing values." >&2 + return 1 + fi + + # Most external components are CGO_ENABLED=0 and run natively on HOST_ARCH. + # vmagent uses CGO_ENABLED=1 and needs gcc matching the target arch (GOARCH). + local container_arch="${HOST_ARCH}" + + echo "Building external component: ${component} (${BUILD_TYPE}, ${GOARCH})" + echo "Repository: ${url}" + echo "Reference: ${ref}" + + mkdir -p "${OUTPUT_DIR}" + + # Determine build command + local build_cmd + case "${component}" in + node_exporter) + build_cmd="make release && cp node_exporter /output/ && mkdir -p /output/queries && cp example.prom /output/queries/" + ;; + mysqld_exporter) + build_cmd="make release && cp mysqld_exporter /output/ && mkdir -p /output/queries && cp queries-mysqld.yml queries-mysqld-group-replication.yml /output/queries/" + ;; + mongodb_exporter) + if [ "${BUILD_TYPE}" = "dynamic" ]; then + build_cmd="make build-gssapi && cp mongodb_exporter /output/" + else + build_cmd="make build && cp mongodb_exporter /output/" + fi + ;; + postgres_exporter) + build_cmd="make release && cp postgres_exporter /output/ && mkdir -p /output/queries && cp example-queries-postgres.yml queries-postgres-uptime.yml queries-hr.yml queries-mr.yaml queries-lr.yaml /output/queries/" + ;; + proxysql_exporter) + build_cmd="make release && cp proxysql_exporter /output/" + ;; + rds_exporter) + build_cmd="make release && cp rds_exporter /output/" + ;; + azure_metrics_exporter) + build_cmd="make release && cp azure_exporter /output/" + ;; + redis_exporter) + build_cmd="make build && cp redis_exporter /output/valkey_exporter" + ;; + vmagent) + build_cmd="make vmagent-linux-amd64 && cp bin/vmagent-linux-amd64 /output/vmagent" + container_arch="${GOARCH}" + ;; + nomad) + build_cmd="env CGO_ENABLED=0 TARGETS=linux_${GOARCH} make deps release && cp pkg/linux_${GOARCH}/nomad /output/" + container_arch="${GOARCH}" + ;; + percona-toolkit) + build_cmd="mkdir -p /output && \ + cp bin/pt-summary /output/ && \ + cp bin/pt-mysql-summary /output/ && \ + cd src/go/pt-mongodb-summary && go build -o /output/pt-mongodb-summary . && \ + cd ../pt-pg-summary && go build -o /output/pt-pg-summary ." + ;; + esac + + # Derive bare repo cache name from URL + # e.g. https://github.com/percona/node_exporter -> node_exporter.git + local bare_name + bare_name="$(basename "${url}")" + [[ "${bare_name}" != *.git ]] && bare_name="${bare_name}.git" + local bare_repo_path="${REPO_CACHE_DIR}/${bare_name}" + + if [ ! -d "${bare_repo_path}" ]; then + echo "Error: bare repo cache not found at ${bare_repo_path}" >&2 + echo "Run 'make populate-cache' (first time) or 'make download-cache' to populate it." >&2 + return 1 + fi + + echo "Using cached bare repo: ${bare_name}" + local clone_and_build=" + set -e + git clone /repos/${bare_name} /build/${component} + cd /build/${component} + git checkout ${ref} + ${build_cmd} + " + docker run --rm \ + --platform "linux/${container_arch}" \ + -v "${bare_repo_path}:/repos/${bare_name}:ro" \ + -v "${OUTPUT_DIR}:/output" \ + -v "${GOMOD_CACHE_VOL}:/go/pkg/mod" \ + -v "${BUILD_CACHE_VOL}:/root/.cache/go-build" \ + -e GOCACHE=/root/.cache/go-build \ + -e CGO_ENABLED=0 \ + -e GOOS=linux \ + -e GOARCH="${GOARCH}" \ + -w /build \ + "${BUILDER_IMAGE}" \ + bash -c "${clone_and_build}" + + echo "Build complete! Artifacts saved to ${OUTPUT_DIR}" + ls -lh "${OUTPUT_DIR}" +} + +# Main +if [ -z "${COMPONENT}" ]; then + usage +fi + +echo "PMM Component Builder" +echo "=====================" +echo "Component: ${COMPONENT}" +echo "Version: ${PMM_VERSION}" +echo "Build Type: ${BUILD_TYPE}" +echo "Architecture: ${GOARCH}" +echo "Platform: ${PLATFORM}" +echo "Output: ${OUTPUT_DIR}" +echo "" + +build_builder_image +create_volumes + +# Check if component is a workspace component +if echo " ${WORKSPACE_COMPONENTS} " | grep -q " ${COMPONENT} "; then + build_workspace_component "${COMPONENT}" +# Check if component is an external component +elif echo " ${EXTERNAL_COMPONENTS} " | grep -q " ${COMPONENT} "; then + build_external_component "${COMPONENT}" +else + echo "Error: unknown component '${COMPONENT}'" + echo "" + echo "Available workspace components: ${WORKSPACE_COMPONENTS}" + echo "Available external components: ${EXTERNAL_COMPONENTS}" + usage +fi diff --git a/build/pipeline/scripts/check-ui-cache b/build/pipeline/scripts/check-ui-cache new file mode 100755 index 00000000000..9e2751677bb --- /dev/null +++ b/build/pipeline/scripts/check-ui-cache @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# check-ui-cache: Determine whether a UI component needs rebuilding. +# +# The check resolves the current ref to a full commit hash (via the local bare repo) +# and compares it against the hash stored in a previously generated VERSION.json. +# If both match AND the output directory is non-empty, the build can be skipped. +# +# Usage: +# check-ui-cache +# +# Arguments: +# json-key Key name inside VERSION.json (e.g. "grafana", "pmm", "pmm-dashboards") +# ref Branch, tag, or commit hash (e.g. value of GRAFANA_REF) +# bare-repo Path to the local bare repo (e.g. .cache/repos/grafana.git) +# output-dir Component output directory (e.g. output/server/grafana-ui) +# version-json Path to existing VERSION.json (e.g. output/server/version.json) +# +# Exit codes: +# 0 — build is needed (cache miss, missing output, or changed hash) +# 1 — build can be skipped (hash unchanged, output already present) + +set -uo pipefail + +COMPONENT="$1" +REF="$2" +BARE_REPO="$3" +OUTPUT_DIR="$4" +VERSION_JSON="$5" + +# Always build if output directory is missing or empty +if [ ! -d "$OUTPUT_DIR" ] || [ -z "$(ls -A "$OUTPUT_DIR" 2>/dev/null)" ]; then + echo "[$COMPONENT] Output dir missing or empty — building." + exit 0 +fi + +# Always build if there is no previous VERSION.json to compare against +if [ ! -f "$VERSION_JSON" ]; then + echo "[$COMPONENT] No previous VERSION.json — building." + exit 0 +fi + +# Resolve the ref to a full commit hash using the local bare repo. +# When REF is empty (not set in .env), fall back to HEAD of the bare repo — +# consistent with what `git checkout` (no argument) does in the build containers. +if [ -d "$BARE_REPO" ]; then + if [ -n "$REF" ]; then + CURRENT_HASH=$(git -C "$BARE_REPO" rev-parse "${REF}^{commit}" 2>/dev/null || echo "") + fi + if [ -z "${CURRENT_HASH:-}" ]; then + CURRENT_HASH=$(git -C "$BARE_REPO" rev-parse HEAD 2>/dev/null || echo "") + fi +else + CURRENT_HASH="$REF" +fi + +if [ -z "$CURRENT_HASH" ]; then + echo "[$COMPONENT] Could not resolve ref '${REF}' — building." + exit 0 +fi + +# Extract the cached hash for this component from VERSION.json +# Uses only grep + sed to avoid requiring python3/jq on the host. +CACHED_HASH=$(grep -o "\"${COMPONENT}\": *\"[^\"]*\"" "$VERSION_JSON" 2>/dev/null \ + | sed 's/.*"\([^"]*\)"$/\1/' \ + || echo "") + +if [ -n "$CACHED_HASH" ] && [ "$CURRENT_HASH" = "$CACHED_HASH" ]; then + echo "[$COMPONENT] Commit hash unchanged (${CURRENT_HASH}) — skipping build." + exit 1 +fi + +echo "[$COMPONENT] Hash changed (current=${CURRENT_HASH}, cached=${CACHED_HASH:-none}) — building." +exit 0 diff --git a/build/pipeline/scripts/generate-version-json b/build/pipeline/scripts/generate-version-json new file mode 100755 index 00000000000..6e596798891 --- /dev/null +++ b/build/pipeline/scripts/generate-version-json @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Generate output/server/version.json with all component git refs. +# +# All *_REF variables and PMM_VERSION are exported by the Makefile before +# this script is called, so they are available in the environment. +# Populate missing values by running 'scripts/migrate-from-submodules' first. +# +# Usage: called via 'make generate-version-json'; not intended for direct invocation. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUTPUT_DIR="${SERVER_OUTPUT_DIR:-$(realpath "${SCRIPT_DIR}/../output/server")}" +OUTPUT_FILE="${OUTPUT_DIR}/version.json" + +REPO_CACHE_DIR="${REPO_CACHE_DIR:-$(realpath "${SCRIPT_DIR}/../.cache/repos")}" + +# get_ref +# Returns the whitespace-stripped value of the variable. +# Exits with an error if the variable is empty. +get_ref() { + local env_var="$1" + local val="${!env_var:-}" + val="$(echo "${val}" | xargs 2>/dev/null || echo "${val}")" + + if [ -z "${val}" ]; then + echo "Error: ${env_var} is not set — run 'scripts/migrate-from-submodules' to populate .env" >&2 + exit 1 + fi + + echo "${val}" +} + +# resolve_commit +# Resolves a branch name, tag, or commit ref to its full commit hash using the +# local bare-repo cache. Falls back to the original ref if resolution fails +# (e.g. repo not yet cached or ref not found). +resolve_commit() { + local ref="$1" + local bare_repo="$2" + local bare_path="${REPO_CACHE_DIR}/${bare_repo}" + + if [ -z "${ref}" ]; then + echo "" + return + fi + + if [ -d "${bare_path}" ]; then + local hash + hash="$(git -C "${bare_path}" rev-parse "${ref}^{commit}" 2>/dev/null || true)" + if [ -n "${hash}" ]; then + echo "${hash}" + return + fi + fi + + # Fallback: already a hash, or repo not cached yet + echo "${ref}" +} + +mkdir -p "${OUTPUT_DIR}" + +PMM_VERSION="${PMM_VERSION:-}" +PMM_VERSION="$(echo "${PMM_VERSION}" | xargs 2>/dev/null || echo "${PMM_VERSION}")" +if [ -z "${PMM_VERSION}" ]; then + echo "Error: PMM_VERSION is not set — run 'scripts/migrate-from-submodules' to populate .env" >&2 + exit 1 +fi + +cat > "${OUTPUT_FILE}" <] +# --branch pmm-submodules branch to read from (default: v3) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${SCRIPT_DIR}/../.env" +BRANCH="v3" + +while [[ $# -gt 0 ]]; do + case "$1" in + --branch) BRANCH="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 1 ;; + esac +done + +SUBMODULES_BASE="https://raw.githubusercontent.com/Percona-Lab/pmm-submodules/refs/heads/${BRANCH}" +GITMODULES_TMP="$(mktemp /tmp/.gitmodules.XXXXXX)" + +trap 'rm -f "${GITMODULES_TMP}"' EXIT + +# set_if_empty +# Writes KEY=VALUE into .env only when the key is absent or currently empty. +set_if_empty() { + local key="$1" + local value="$2" + + if grep -qE "^${key}=.+" "${ENV_FILE}" 2>/dev/null; then + printf ' %-40s already set, skipping\n' "${key}" + return + fi + + if grep -qE "^${key}=" "${ENV_FILE}" 2>/dev/null; then + # Key present but empty — update in-place using a temp file (portable). + local tmp + tmp="$(mktemp)" + sed "s|^${key}=$|${key}=${value}|" "${ENV_FILE}" > "${tmp}" + mv "${tmp}" "${ENV_FILE}" + printf ' %-40s set to %s\n' "${key}" "${value}" + else + # Key absent — append it. + echo "${key}=${value}" >> "${ENV_FILE}" + printf ' %-40s added (%s)\n' "${key}" "${value}" + fi +} + +# parse_gitmodules +# Prints one line per submodule: nameurlref +# ref is the branch value; falls back to tag if branch is absent. +parse_gitmodules() { + local file="$1" + awk ' + function emit() { + if (name != "" && url != "") { + print name "\t" url "\t" ref + } + } + /^\[submodule / { + emit() + name = $0 + gsub(/^\[submodule "/, "", name) + gsub(/".*$/, "", name) + url = ""; ref = "" + } + /url[ \t]*=/ { + sub(/^[^=]*=[ \t]*/, "") + url = $0 + } + /branch[ \t]*=/ { + sub(/^[^=]*=[ \t]*/, "") + ref = $0 + } + /tag[ \t]*=/ { + if (ref == "") { + sub(/^[^=]*=[ \t]*/, "") + ref = $0 + } + } + END { emit() } + ' "${file}" +} + +# ── Step 1: PMM_VERSION ────────────────────────────────────────────────────── +echo "Fetching PMM_VERSION from pmm-submodules/${BRANCH}/VERSION ..." +PMM_VERSION="$(curl -fsSL "${SUBMODULES_BASE}/VERSION" | tr -d '[:space:]')" +if [ -z "${PMM_VERSION}" ]; then + echo "ERROR: Could not fetch VERSION from pmm-submodules" >&2 + exit 1 +fi +echo " Found: ${PMM_VERSION}" +set_if_empty "PMM_VERSION" "${PMM_VERSION}" + +# ── Step 2: Fetch .gitmodules ──────────────────────────────────────────────── +echo "" +echo "Fetching .gitmodules from pmm-submodules/${BRANCH}/.gitmodules ..." +curl -fsSL "${SUBMODULES_BASE}/.gitmodules" -o "${GITMODULES_TMP}" +echo " Done ($(wc -l < "${GITMODULES_TMP}") lines)" + +# ── Step 3: Apply values from .gitmodules ──────────────────────────────────── +# +# Maps each submodule name as it appears in .gitmodules to the env-var prefix +# used in .env (e.g. "grafana" → GRAFANA). +declare -A NAME_TO_PREFIX=( + ["pmm"]="PMM" + ["grafana"]="GRAFANA" + ["VictoriaMetrics"]="VM" +["pmm-dump"]="PMM_DUMP" + ["node_exporter"]="NODE_EXPORTER" + ["mysqld_exporter"]="MYSQLD_EXPORTER" + ["mongodb_exporter"]="MONGODB_EXPORTER" + ["postgres_exporter"]="POSTGRES_EXPORTER" + ["proxysql_exporter"]="PROXYSQL_EXPORTER" + ["rds_exporter"]="RDS_EXPORTER" + ["azure_metrics_exporter"]="AZURE_METRICS_EXPORTER" + ["percona-toolkit"]="PERCONA_TOOLKIT" +) + +echo "" +echo "Processing .gitmodules entries ..." +while IFS=$'\t' read -r name url ref; do + prefix="${NAME_TO_PREFIX["${name}"]:-}" + if [ -z "${prefix}" ]; then + echo " Skipping unknown submodule: ${name}" + continue + fi + set_if_empty "${prefix}_URL" "${url}" + if [ -n "${ref}" ]; then + set_if_empty "${prefix}_REF" "${ref}" + fi +done < <(parse_gitmodules "${GITMODULES_TMP}") + +# ── Step 4: Default UI/dashboards refs to PMM_REF ─────────────────────────── +# PMM_UI_REF and PMM_DASHBOARDS_REF are not separate submodules — they come from +# the same percona/pmm repo as PMM_REF. Seed them with the same ref unless +# already overridden. +echo "" +echo "Setting PMM_UI_REF and PMM_DASHBOARDS_REF (default to PMM_REF) ..." +_PMM_REF="$(grep -E '^PMM_REF=.+' "${ENV_FILE}" | head -1 | cut -d= -f2-)" +if [ -n "${_PMM_REF}" ]; then + set_if_empty "PMM_UI_REF" "${_PMM_REF}" + set_if_empty "PMM_DASHBOARDS_REF" "${_PMM_REF}" +else + echo " PMM_REF not yet set — skipping PMM_UI_REF / PMM_DASHBOARDS_REF seeding" +fi + +# ── Step 5: Hardcoded URLs for components absent from .gitmodules ──────────── +echo "" +echo "Setting URLs for components not in .gitmodules ..." +set_if_empty "REDIS_EXPORTER_URL" "https://github.com/oliver006/redis_exporter.git" +set_if_empty "VMAGENT_URL" "https://github.com/VictoriaMetrics/VictoriaMetrics.git" +set_if_empty "NOMAD_URL" "https://github.com/hashicorp/nomad.git" + +echo "" +echo "Migration complete." +echo "Review ${ENV_FILE} and commit the result." +echo "You can now remove the pmm-submodules references from the build scripts." diff --git a/build/pipeline/scripts/package-tarball b/build/pipeline/scripts/package-tarball new file mode 100755 index 00000000000..4538bd2cb1a --- /dev/null +++ b/build/pipeline/scripts/package-tarball @@ -0,0 +1,142 @@ +#!/bin/bash +# Package PMM client tarball + +set -o errexit +set -o pipefail +set -o nounset + +# Default values +SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +PIPELINE_DIR="$(dirname "${SCRIPT_DIR}")" +WORKSPACE_DIR="$(realpath "$SCRIPT_DIR/../../..")" +PMM_VERSION="${PMM_VERSION:?No PMM_VERSION specified}" +OUTPUT_DIR="${OUTPUT_DIR:-${PIPELINE_DIR}/output}" +PACKAGE_DIR="${PACKAGE_DIR:-${PIPELINE_DIR}/package}" + +# Package name +PACKAGE_NAME="pmm-client-${PMM_VERSION}" +PACKAGE_ROOT="${PACKAGE_DIR}/${PACKAGE_NAME}" + +usage() { + cat <&2 + exit 1 + fi +done + +echo "${PMM_VERSION}" > "${PACKAGE_ROOT}/VERSION" + +# Copy install_tarball script +echo "Copying install_tarball script..." +cp "${WORKSPACE_DIR}/build/scripts/install_tarball" "${PACKAGE_ROOT}/" +chmod +x "${PACKAGE_ROOT}/install_tarball" + +echo "Creating tarball..." +declare XATTRS="" +[ "$(uname -s)" = "Darwin" ] && XATTRS="--no-xattrs" +tar -czf "${PACKAGE_DIR}/${PACKAGE_NAME}.tar.gz" "$XATTRS" -C "${PACKAGE_DIR}" "${PACKAGE_NAME}" + +echo "" +echo "Package created successfully!" +echo "Location: ${PACKAGE_DIR}/${PACKAGE_NAME}.tar.gz" +ls -lh "${PACKAGE_DIR}/${PACKAGE_NAME}.tar.gz" diff --git a/go.mod b/go.mod index 90e592991a8..0f319444c9f 100644 --- a/go.mod +++ b/go.mod @@ -74,6 +74,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 google.golang.org/grpc v1.80.0 google.golang.org/protobuf v1.36.11 + gopkg.in/ini.v1 v1.67.1 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 gopkg.in/reform.v1 v1.5.1 gopkg.in/yaml.v3 v3.0.1 @@ -131,7 +132,6 @@ require ( go.opentelemetry.io/otel/metric v1.41.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect - gopkg.in/ini.v1 v1.67.1 // indirect ) require ( diff --git a/managed/Makefile b/managed/Makefile index 80c28b3ebc9..d35d5521cf9 100644 --- a/managed/Makefile +++ b/managed/Makefile @@ -12,6 +12,7 @@ PMM_RELEASE_VERSION ?= $(shell git describe --always | cut -b2-) PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s') PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD) PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all) +GOARCH ?= amd64 PMM_LD_FLAGS = -ldflags " \ -X 'github.com/percona/pmm/version.ProjectName=pmm-managed' \ @@ -39,13 +40,13 @@ pi-validate: ## Validate Percona Intelligence advisors and checks go run ./cmd/pi-validator/main.go advisors --advisors.dir ./data/advisors/ --checks.dir ./data/checks/ release: ## Build pmm-managed release binaries - env CGO_ENABLED=0 go build -v $(PMM_LD_FLAGS) -o $(PMM_RELEASE_PATH)/ ./cmd/... + env CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -v $(PMM_LD_FLAGS) -o $(PMM_RELEASE_PATH)/ ./cmd/... release-encryption-rotation: ## Build PMM encryption rotation tool - env CGO_ENABLED=0 go build -v $(PMM_LD_FLAGS) -o $(PMM_RELEASE_PATH)/ ./cmd/pmm-encryption-rotation/... + env CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -v $(PMM_LD_FLAGS) -o $(PMM_RELEASE_PATH)/ ./cmd/pmm-encryption-rotation/... release-starlark: - env CGO_ENABLED=0 go build -v $(PMM_LD_FLAGS) -o $(PMM_RELEASE_PATH)/ ./cmd/pmm-managed-starlark/... + env CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -v $(PMM_LD_FLAGS) -o $(PMM_RELEASE_PATH)/ ./cmd/pmm-managed-starlark/... $(PMM_RELEASE_PATH)/pmm-managed-starlark --version release-dev: ## Build pmm-managed binaries for development diff --git a/qan-api2/Makefile b/qan-api2/Makefile index 6a1c9104ea2..d6d5c209808 100644 --- a/qan-api2/Makefile +++ b/qan-api2/Makefile @@ -14,11 +14,12 @@ PMM_RELEASE_VERSION ?= $(shell git describe --always --dirty | cut -b2-) PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s') PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD) PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all) +GOARCH ?= amd64 PMM_CONTAINER ?= pmm-server release: ## Build qan-api2 release binary - env CGO_ENABLED=0 go build -v -o $(PMM_RELEASE_PATH)/qan-api2 -ldflags " \ + env CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -v -o $(PMM_RELEASE_PATH)/qan-api2 -ldflags " \ -X 'github.com/percona/pmm/version.ProjectName=qan-api2' \ -X 'github.com/percona/pmm/version.Version=$(PMM_RELEASE_VERSION)' \ -X 'github.com/percona/pmm/version.PMMVersion=$(PMM_RELEASE_VERSION)' \ diff --git a/vmproxy/Makefile b/vmproxy/Makefile index de22f04d719..1b92b2a79c5 100644 --- a/vmproxy/Makefile +++ b/vmproxy/Makefile @@ -11,6 +11,7 @@ PMM_RELEASE_VERSION ?= $(shell git describe --always --dirty | cut -b2-) PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s') PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD) PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all) +GOARCH ?= amd64 ifeq ($(GOBIN),) GOBIN := $(shell go env GOPATH)/bin endif @@ -25,7 +26,7 @@ LD_FLAGS = -ldflags " \ " release: ## Build vmproxy release binary - env CGO_ENABLED=0 go build -v $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/vmproxy . + env CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -v $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/vmproxy . install: ## Install vmproxy binary go build -v $(LD_FLAGS) -o $(GOBIN)/vmproxy .