PMM-14887 Restructure Makefiles

.devcontainer/Dockerfile

@@ -14,8 +14,6 @@ RUN mkdir -p $GOPATH/src/github.com/percona/pmm
 WORKDIR $GOPATH/src/github.com/percona/pmm
 
 COPY ./ ./
-# setup.py uses a task from Makefile.devcontainer but expects it to be in the default Makefile
-COPY ./Makefile.devcontainer ./Makefile
 
 RUN --mount=type=cache,target=/var/cache/dnf \
     python ./.devcontainer/setup.py

Makefile.devcontainer → .devcontainer/Makefile

@@ -1,7 +1,5 @@
 # Devcontainer Makefile.
 
-include Makefile.include
-
 release-dev-managed:              ## Build pmm-managed
 	make -C managed release-dev
 
@@ -20,7 +18,7 @@ _bash:
 
 PMM_RELEASE_PATH ?= ./bin
 
-run-managed-ci: release-dev-managed    ## Replace pmm-managed from build, restart (used in CI)	supervisorctl stop pmm-managed
+run-managed-ci: release-dev-managed    ## Replace pmm-managed from build, restart (used in CI)
 	supervisorctl stop pmm-managed
 	truncate -s 0 /srv/logs/pmm-managed.log
 	cp $(PMM_RELEASE_PATH)/pmm-managed /usr/sbin/pmm-managed
@@ -58,9 +56,6 @@ run-qan: run-qan-ci	                ## Replace qan-api2 from build, restart and
 
 run-all: run-agent-ci run-managed-ci run-qan-ci run-vmproxy-ci	## Run all go services
 
-run: run-all                         ## Aliased to "run-all"
-	echo "run is aliased to run-all"
-
 # TODO https://jira.percona.com/browse/PMM-3484, see maincover_test.go
 # run-race-cover: install-race    ## Run pmm-managed with race detector and collect coverage information.
 # 	go test -coverpkg="github.com/percona/pmm/managed/..." \
@@ -69,13 +64,13 @@ run: run-all                         ## Aliased to "run-all"
 # 			-race -c -o bin/pmm-managed.test
 # 	bin/pmm-managed.test -test.coverprofile=cover.out -test.run=TestMainCover $(RUN_FLAGS)
 
-psql:                           ## Open database for the pmm-managed instance in psql shell
+psql-managed:                   ## Open database for the pmm-managed instance in psql shell
 	env PGPASSWORD=pmm-managed psql -U pmm-managed pmm-managed
 
-psql-test:                      ## Open database used in unit tests in psql shell
-	env psql -U postgres pmm-managed-dev
+psql-managed-dev:               ## Open database used in unit tests in psql shell
+	psql -U postgres pmm-managed-dev
 
-dlv/attach:                     ## Attach Delve to `pmm-managed`
+dlv-attach:                     ## Attach Delve to `pmm-managed`
 	dlv --listen=:2345 --headless=true --api-version=2 --continue=true --accept-multiclient attach $(shell pgrep pmm-managed)
 
 refresh-swagger:                ## Refresh swagger files

.env.example

@@ -1,6 +1,17 @@
-# Percona PMM dev container
-#PMM_CONTAINER=perconalab/pmm-server:3-dev-container
+# This is a sample .env file for PMM Server. You can copy this file to .env and modify the values as needed.
 
-CH_IMAGE=clickhouse/clickhouse-server:22.6.9.11-alpine
-CH_PORT=9000
-CH_HOSTNAME=ch
+# The latest PMM Server image is alsways published as percona/pmm-server:3.
+# A specific version can be used by changing the tag, e.g., percona/pmm-server:3.7.0.
+PMM_SERVER_IMAGE=percona/pmm-server:3
+# Shared docker network for PMM Server and its components.
+PMM_NETWORK=pmm-network
+# Label-based access control for PMM Server. Set to 1 to enable.
+PMM_ENABLE_ACCESS_CONTROL=1
+# Good for development, but not recommended for production environments.
+PMM_ENABLE_INTERNAL_PG_QAN=1
+# Controls the log verbosity of PMM Server. Set to 1 to enable debug logging, which can be helpful for troubleshooting.
+PMM_DEBUG=0
+# Port mapping for PMM Server. The default is 443, but you can change it if needed.
+PMM_PORT_HTTPS=443
+# Change the admin password for PMM Server. The default is "admin", but you should set a strong password for production environments.
+PMM_ADMIN_PASSWORD=admin

.github/instructions/managed.instructions.md

@@ -3,8 +3,11 @@ applyTo: managed/**
 ---
 # pmm-managed Development Guidelines
 
-> **Parent guide**: [AGENTS.md](../../AGENTS.md) — product overview, architecture, domain model, global conventions
-> **Related**: [api.instructions.md](api.instructions.md) (API definitions) · [agent.instructions.md](agent.instructions.md) (client agent) · [qan-api2.instructions.md](qan-api2.instructions.md) (QAN backend)
+- **Parent guide**: [AGENTS.md](../../AGENTS.md) — product overview, architecture, domain model, global conventions
+- **Related**: 
+  - [api.instructions.md](api.instructions.md) (API definitions) 
+  - [agent.instructions.md](agent.instructions.md) (client agent) 
+  - [qan-api2.instructions.md](qan-api2.instructions.md) (QAN backend)
 
 **pmm-managed** is the core backend service of PMM Server. It manages configuration of server-side components (VictoriaMetrics, Grafana, QAN, VMAlert, Alertmanager), maintains the inventory of monitored nodes/services/agents, orchestrates backups, runs advisor checks, handles HA consensus, and exposes gRPC/REST APIs consumed by pmm-admin, pmm-agent, and the UI.
 
@@ -177,7 +180,7 @@ PMM supports HA via **Raft consensus** (`services/ha/`):
 
 ## Code Generation
 
-1. **Protocol Buffers** — `make gen` from repo root
+1. **Protocol Buffers** — `make gen`, followed by `make format` (from project root)
 2. **reform** — `//go:generate ../../bin/reform` on model files
 3. **mockery** — mock generation per `.mockery.yaml`
 4. **swagger** — API docs from proto annotations
@@ -194,5 +197,5 @@ Always run `make gen` after modifying `.proto` files, reform models, or interfac
 - `managed/services/inventory/grpc/` — inventory API implementations
 - `managed/services/ha/` — HA/Raft implementation
 - `managed/utils/envvars/parser.go` — environment variable parsing
-- `docker-compose.yml` — development environment
-- `Makefile`, `Makefile.include` — build and development targets
+- `docker-compose.dev.yml` — development environment
+- `Makefile` — build and development targets

.github/workflows/api-tests.yml

@@ -67,7 +67,6 @@ jobs:
                 - PMM_DEBUG=1
                 - PMM_ENABLE_TELEMETRY=0
                 - PMM_DEV_PERCONA_PLATFORM_ADDRESS=https://check-dev.percona.com
-                - PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY=RWTg+ZmCCjt7O8eWeAmTLAqW+1ozUbpRSKSwNTmO+exlS5KEIPYWuYdX
               ports:
                 - 443:8443
               volumes:

.github/workflows/main.yml

@@ -131,21 +131,8 @@ jobs:
           go env | sort
           git status
 
-
-  spell-check:
-    name: Spell check
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Check spelling of md files
-        uses: crate-ci/typos@02ea592e44b3a53c302f697cddca7641cd051c3d # v1.45.0
-        with:
-          files: "**/*.md ./documentation/**/*.md"
-
   merge-gatekeeper:
-    needs: [ check, spell-check ]
+    needs: [ check ]
     name: Merge Gatekeeper
     if: ${{ always() }}
     runs-on: ubuntu-22.04

.github/workflows/managed.yml

@@ -28,17 +28,15 @@ permissions: read-all
 
 jobs:
   test:
-    name: Managed tests
+    name: Unit tests
     runs-on: ubuntu-22.04
     timeout-minutes: 30
 
     env:
-      PMM_SERVER_IMAGE: perconalab/pmm-server:3-dev-latest
+      # PMM_SERVER_IMAGE: perconalab/pmm-server:3-dev-container
+      PMM_SERVER_IMAGE: ghcr.io/percona/pmm:3-dev-container
       AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
       AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
-      PMM_DEV_OAUTH_CLIENT_ID: ${{ secrets.OAUTH_PMM_CLIENT_ID }}
-      PMM_DEV_OAUTH_CLIENT_SECRET: ${{ secrets.OAUTH_PMM_CLIENT_SECRET }}
-      PMM_CONTAINER: ghcr.io/percona/pmm:3-dev-container
 
     steps:
       - name: Check out code
@@ -66,7 +64,7 @@ jobs:
             -e PMM_RELEASE_PATH=/root/go/bin \
             -e PMM_ENABLE_ACCESS_CONTROL=1 \
             -e PMM_ENABLE_TELEMETRY=0 \
-            -e PMM_RELEASE_VERSION=3.0.0 \
+            -e PMM_RELEASE_VERSION=3.7.0 \
             -e GOMODCACHE=/home/pmm/.cache/go/mod \
             -e GOCACHE=/home/pmm/.cache/go-build \
             -w /root/go/src/github.com/percona/pmm \
@@ -76,8 +74,7 @@ jobs:
             -v ./managed/data/advisors/:/usr/local/percona/advisors/ \
             -v ./managed/data/checks/:/usr/local/percona/checks/ \
             -v ./managed/data/alerting-templates/:/usr/local/percona/alerting-templates/ \
-            -v ./Makefile.devcontainer:/root/go/src/github.com/percona/pmm/Makefile:ro \
-            ${PMM_CONTAINER:-perconalab/pmm-server:3-dev-container}
+            ${PMM_SERVER_IMAGE}
           sleep 20
           docker exec -t pmm-server id
           docker logs pmm-server

AGENTS.md

@@ -230,7 +230,8 @@ All long-running daemons expose on `127.0.0.1`:
 ## Key Files to Reference
 
 - `Makefile`, `Makefile.include` — build and development targets
-- `docker-compose.yml` — development environment (PMM Server, renderer, watchtower)
+- `docker-compose.dev.yml` — development environment (PMM Server, renderer, watchtower)
+- `docker-compose.yml` — community/quickstart compose (stable image, minimal config)
 - `go.mod` — Go module definition
 - `.golangci.yml` — linter configuration
 - `.mockery.yaml` — mock generation configuration

Makefile

@@ -1,48 +1,125 @@
-# Host Makefile.
+# Development Makefile
+.PHONY: default help init release gen test clean all
+.DEFAULT_GOAL := help
 
-include Makefile.include
 -include documentation/Makefile
+-include .devcontainer/Makefile
 
-ifeq ($(PROFILES),)
-PROFILES := 'pmm'
-endif
+default: help
 
-env-up: 							## Start devcontainer
+help:                 ## Display this help message
+	@echo "Please use \`make <target>\`, where <target> is one of the following:"
+	@grep -h '^[a-zA-Z]' $(MAKEFILE_LIST) | awk -F ':.*## ' 'NF==2 {printf "  %-26s%s\n", $$1, $$2}' | sort
+	@echo
+	@echo For developers: check docker-compose.dev.yml to see which environment variables are available.
+
+init:                 ## Install tools
+	rm -rf bin/*
+	cd tools && go generate -x -tags=tools
+
+	# Install golangci-lint
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./bin v2.6.2 # Version should match specified in CI
+
+
+PROFILES ?= pmm
+COMPOSE_FILE ?= docker-compose.dev.yml
+
+env-up:               ## Start devcontainer
 	COMPOSE_PROFILES=$(PROFILES) \
-	docker compose up -d --wait --wait-timeout 100
+	docker compose -f $(COMPOSE_FILE) up -d --wait --wait-timeout 100
 
 env-up-rebuild: env-update-image	## Rebuild and start devcontainer. Useful for custom $PMM_SERVER_IMAGE
 	COMPOSE_PROFILES=$(PROFILES) \
-	docker compose up --build -d
+	docker compose -f $(COMPOSE_FILE) up --build -d
 
-env-update-image:					## Pull latest dev image
+env-update-image:     ## Pull latest dev image
 	COMPOSE_PROFILES=$(PROFILES) \
-	docker compose pull
+	docker compose -f $(COMPOSE_FILE) pull
 
-env-compose-up: env-update-image
+env-compose-up: env-update-image  ## Pull the image, then start devcontainer waiting for it to be ready
 	COMPOSE_PROFILES=$(PROFILES) \
-	docker compose up --detach --renew-anon-volumes --remove-orphans --wait --wait-timeout 100
+	docker compose -f $(COMPOSE_FILE) up -d --renew-anon-volumes --remove-orphans --wait --wait-timeout 100
 
-env-devcontainer:
+env-devcontainer:     ## Provision devcontainer (run this after `make env-up` or `make env-compose-up`)
 	docker exec -it --workdir=/root/go/src/github.com/percona/pmm --user root pmm-server python .devcontainer/setup.py
 
-env-down:							## Stop devcontainer
+env-down:             ## Stop devcontainer
 	COMPOSE_PROFILES=$(PROFILES) \
-	docker compose down --remove-orphans
+	docker compose -f $(COMPOSE_FILE) down --remove-orphans
 
-env-remove:
+env-remove:           ## Stop devcontainer and remove volumes
 	COMPOSE_PROFILES=$(PROFILES) \
-	docker compose down --volumes --remove-orphans
+	docker compose -f $(COMPOSE_FILE) down --volumes --remove-orphans
 
 TARGET ?= _bash
 
-env:								## Run `make TARGET` in devcontainer (`make env TARGET=help`); TARGET defaults to bash
-	COMPOSE_PROFILES=$(PROFILES) \
+env:									## Run `make TARGET` in devcontainer (`make env TARGET=help`); TARGET defaults to bash
 	docker exec -it --workdir=/root/go/src/github.com/percona/pmm pmm-server make $(TARGET)
 
-env-root:								## Run `make TARGET` in devcontainer (`make env-root TARGET=help`); TARGET defaults to bash
-	COMPOSE_PROFILES=$(PROFILES) \
+env-root:             ## Run `make TARGET` in devcontainer (`make env-root TARGET=help`); TARGET defaults to bash
 	docker exec -it --workdir=/root/go/src/github.com/percona/pmm --user root pmm-server make $(TARGET)
 
-rotate-encryption: 							## Rotate encryption key
+rotate-encryption:    ## Rotate encryption key
 	go run ./encryption-rotation/main.go
+
+release:              ## Build release versions of all components
+	make -C agent release
+	make -C admin release
+	make -C managed release
+	make -C qan-api2 release
+
+gen: clean            ## Generate files
+	make -C api gen
+	make -C api clean-swagger
+
+	make -C agent gen
+	make -C admin gen
+	make -C managed gen
+
+	make gen-mocks      ## Generate mocks
+
+	make format
+	make format ## TODO: One formatting run is not enough, figure out why.
+	go install -v ./...
+
+clean:                ## Remove generated files
+	make -C api clean
+
+gen-mocks:						## Generate mocks for API
+	find . -name mock_*.go -delete
+	./bin/mockery --config .mockery.yaml
+
+test-common:          ## Run tests from API (and other shared) packages only (i.e it ignores directories that are explicitly listed)
+	go test $(shell go list ./... | grep -v -e admin -e agent -e managed -e api-tests -e qan-api2 -e update)
+
+api-test:             ## Run API tests on dev env.
+	go test -count=1 -race -p 1 -v ./api-tests/... -pmm.server-insecure-tls
+
+GOLANG_CI_LINT_RUN_OPTS ?=
+check:                ## Run required checks and linters
+	bin/buf lint -v api
+	LOG_LEVEL=error bin/golangci-lint run -c=.golangci.yml --new-from-rev=$(shell git merge-base v3 HEAD) --new $(GOLANG_CI_LINT_RUN_OPTS)
+	bin/go-sumtype ./...
+
+check-license:        ## Run license header checks against source files
+	bin/license-eye -c .licenserc.yaml header check
+
+check-all: check-license check    ## Run linter and license checks
+
+check-new:           ## Run linters only against new code since v3 branch point
+	bin/golangci-lint run -c=.golangci.yml --new-from-rev=origin/v3 --new
+
+FILES = $(shell find . -type f -name '*.go')
+
+format:               ## Format source code
+	make -C api format
+	bin/gofumpt -l -w $(FILES)
+	bin/goimports -local github.com/percona/pmm -l -w $(FILES)
+	bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES)
+
+serve:                ## Serve API documentation with nginx
+	nginx -p . -c api/nginx/nginx.conf
+
+GOLANG_CI_LINT_RUN_OPTS=--fix
+prepare-pr: gen check-all  ## Run all checks and generate files
+	go mod tidy