diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 9ad8fcea189..c09d0591e8b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -14,8 +14,6 @@ RUN mkdir -p $GOPATH/src/github.com/percona/pmm WORKDIR $GOPATH/src/github.com/percona/pmm COPY ./ ./ -# setup.py uses a task from Makefile.devcontainer but expects it to be in the default Makefile -COPY ./Makefile.devcontainer ./Makefile RUN --mount=type=cache,target=/var/cache/dnf \ python ./.devcontainer/setup.py diff --git a/Makefile.devcontainer b/.devcontainer/Makefile similarity index 87% rename from Makefile.devcontainer rename to .devcontainer/Makefile index 0ea8ec84bb9..78be0e45200 100644 --- a/Makefile.devcontainer +++ b/.devcontainer/Makefile @@ -1,7 +1,5 @@ # Devcontainer Makefile. -include Makefile.include - release-dev-managed: ## Build pmm-managed make -C managed release-dev @@ -20,7 +18,7 @@ _bash: PMM_RELEASE_PATH ?= ./bin -run-managed-ci: release-dev-managed ## Replace pmm-managed from build, restart (used in CI) supervisorctl stop pmm-managed +run-managed-ci: release-dev-managed ## Replace pmm-managed from build, restart (used in CI) supervisorctl stop pmm-managed truncate -s 0 /srv/logs/pmm-managed.log cp $(PMM_RELEASE_PATH)/pmm-managed /usr/sbin/pmm-managed @@ -58,9 +56,6 @@ run-qan: run-qan-ci ## Replace qan-api2 from build, restart and run-all: run-agent-ci run-managed-ci run-qan-ci run-vmproxy-ci ## Run all go services -run: run-all ## Aliased to "run-all" - echo "run is aliased to run-all" - # TODO https://jira.percona.com/browse/PMM-3484, see maincover_test.go # run-race-cover: install-race ## Run pmm-managed with race detector and collect coverage information. # go test -coverpkg="github.com/percona/pmm/managed/..." \ @@ -69,13 +64,13 @@ run: run-all ## Aliased to "run-all" # -race -c -o bin/pmm-managed.test # bin/pmm-managed.test -test.coverprofile=cover.out -test.run=TestMainCover $(RUN_FLAGS) -psql: ## Open database for the pmm-managed instance in psql shell +psql-managed: ## Open database for the pmm-managed instance in psql shell env PGPASSWORD=pmm-managed psql -U pmm-managed pmm-managed -psql-test: ## Open database used in unit tests in psql shell - env psql -U postgres pmm-managed-dev +psql-managed-dev: ## Open database used in unit tests in psql shell + psql -U postgres pmm-managed-dev -dlv/attach: ## Attach Delve to `pmm-managed` +dlv-attach: ## Attach Delve to `pmm-managed` dlv --listen=:2345 --headless=true --api-version=2 --continue=true --accept-multiclient attach $(shell pgrep pmm-managed) refresh-swagger: ## Refresh swagger files diff --git a/.env.example b/.env.example index 23c5ba92fce..e62af990dff 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,17 @@ -# Percona PMM dev container -#PMM_CONTAINER=perconalab/pmm-server:3-dev-container +# This is a sample .env file for PMM Server. You can copy this file to .env and modify the values as needed. -CH_IMAGE=clickhouse/clickhouse-server:22.6.9.11-alpine -CH_PORT=9000 -CH_HOSTNAME=ch +# The latest PMM Server image is alsways published as percona/pmm-server:3. +# A specific version can be used by changing the tag, e.g., percona/pmm-server:3.7.0. +PMM_SERVER_IMAGE=percona/pmm-server:3 +# Shared docker network for PMM Server and its components. +PMM_NETWORK=pmm-network +# Label-based access control for PMM Server. Set to 1 to enable. +PMM_ENABLE_ACCESS_CONTROL=1 +# Good for development, but not recommended for production environments. +PMM_ENABLE_INTERNAL_PG_QAN=1 +# Controls the log verbosity of PMM Server. Set to 1 to enable debug logging, which can be helpful for troubleshooting. +PMM_DEBUG=0 +# Port mapping for PMM Server. The default is 443, but you can change it if needed. +PMM_PORT_HTTPS=443 +# Change the admin password for PMM Server. The default is "admin", but you should set a strong password for production environments. +PMM_ADMIN_PASSWORD=admin diff --git a/.github/instructions/managed.instructions.md b/.github/instructions/managed.instructions.md index 06c0675d33b..2673b1a19bf 100644 --- a/.github/instructions/managed.instructions.md +++ b/.github/instructions/managed.instructions.md @@ -3,8 +3,11 @@ applyTo: managed/** --- # pmm-managed Development Guidelines -> **Parent guide**: [AGENTS.md](../../AGENTS.md) — product overview, architecture, domain model, global conventions -> **Related**: [api.instructions.md](api.instructions.md) (API definitions) · [agent.instructions.md](agent.instructions.md) (client agent) · [qan-api2.instructions.md](qan-api2.instructions.md) (QAN backend) +- **Parent guide**: [AGENTS.md](../../AGENTS.md) — product overview, architecture, domain model, global conventions +- **Related**: + - [api.instructions.md](api.instructions.md) (API definitions) + - [agent.instructions.md](agent.instructions.md) (client agent) + - [qan-api2.instructions.md](qan-api2.instructions.md) (QAN backend) **pmm-managed** is the core backend service of PMM Server. It manages configuration of server-side components (VictoriaMetrics, Grafana, QAN, VMAlert, Alertmanager), maintains the inventory of monitored nodes/services/agents, orchestrates backups, runs advisor checks, handles HA consensus, and exposes gRPC/REST APIs consumed by pmm-admin, pmm-agent, and the UI. @@ -177,7 +180,7 @@ PMM supports HA via **Raft consensus** (`services/ha/`): ## Code Generation -1. **Protocol Buffers** — `make gen` from repo root +1. **Protocol Buffers** — `make gen`, followed by `make format` (from project root) 2. **reform** — `//go:generate ../../bin/reform` on model files 3. **mockery** — mock generation per `.mockery.yaml` 4. **swagger** — API docs from proto annotations @@ -194,5 +197,5 @@ Always run `make gen` after modifying `.proto` files, reform models, or interfac - `managed/services/inventory/grpc/` — inventory API implementations - `managed/services/ha/` — HA/Raft implementation - `managed/utils/envvars/parser.go` — environment variable parsing -- `docker-compose.yml` — development environment -- `Makefile`, `Makefile.include` — build and development targets +- `docker-compose.dev.yml` — development environment +- `Makefile` — build and development targets diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 4e480a9d615..a8f5acea75c 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -67,7 +67,6 @@ jobs: - PMM_DEBUG=1 - PMM_ENABLE_TELEMETRY=0 - PMM_DEV_PERCONA_PLATFORM_ADDRESS=https://check-dev.percona.com - - PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY=RWTg+ZmCCjt7O8eWeAmTLAqW+1ozUbpRSKSwNTmO+exlS5KEIPYWuYdX ports: - 443:8443 volumes: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5d6061a094f..afcd4672b36 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -131,21 +131,8 @@ jobs: go env | sort git status - - spell-check: - name: Spell check - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Check spelling of md files - uses: crate-ci/typos@02ea592e44b3a53c302f697cddca7641cd051c3d # v1.45.0 - with: - files: "**/*.md ./documentation/**/*.md" - merge-gatekeeper: - needs: [ check, spell-check ] + needs: [ check ] name: Merge Gatekeeper if: ${{ always() }} runs-on: ubuntu-22.04 diff --git a/.github/workflows/managed.yml b/.github/workflows/managed.yml index 20997ce9afb..5beba3fe62b 100644 --- a/.github/workflows/managed.yml +++ b/.github/workflows/managed.yml @@ -28,17 +28,15 @@ permissions: read-all jobs: test: - name: Managed tests + name: Unit tests runs-on: ubuntu-22.04 timeout-minutes: 30 env: - PMM_SERVER_IMAGE: perconalab/pmm-server:3-dev-latest + # PMM_SERVER_IMAGE: perconalab/pmm-server:3-dev-container + PMM_SERVER_IMAGE: ghcr.io/percona/pmm:3-dev-container AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }} AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} - PMM_DEV_OAUTH_CLIENT_ID: ${{ secrets.OAUTH_PMM_CLIENT_ID }} - PMM_DEV_OAUTH_CLIENT_SECRET: ${{ secrets.OAUTH_PMM_CLIENT_SECRET }} - PMM_CONTAINER: ghcr.io/percona/pmm:3-dev-container steps: - name: Check out code @@ -66,7 +64,7 @@ jobs: -e PMM_RELEASE_PATH=/root/go/bin \ -e PMM_ENABLE_ACCESS_CONTROL=1 \ -e PMM_ENABLE_TELEMETRY=0 \ - -e PMM_RELEASE_VERSION=3.0.0 \ + -e PMM_RELEASE_VERSION=3.7.0 \ -e GOMODCACHE=/home/pmm/.cache/go/mod \ -e GOCACHE=/home/pmm/.cache/go-build \ -w /root/go/src/github.com/percona/pmm \ @@ -76,8 +74,7 @@ jobs: -v ./managed/data/advisors/:/usr/local/percona/advisors/ \ -v ./managed/data/checks/:/usr/local/percona/checks/ \ -v ./managed/data/alerting-templates/:/usr/local/percona/alerting-templates/ \ - -v ./Makefile.devcontainer:/root/go/src/github.com/percona/pmm/Makefile:ro \ - ${PMM_CONTAINER:-perconalab/pmm-server:3-dev-container} + ${PMM_SERVER_IMAGE} sleep 20 docker exec -t pmm-server id docker logs pmm-server diff --git a/.typos.toml b/.typos.toml deleted file mode 100644 index 2f7002d330a..00000000000 --- a/.typos.toml +++ /dev/null @@ -1,6 +0,0 @@ -[default] -extend-ignore-identifiers-re = [ - "LKE", - "ue", - "ba" -] \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index f1629fcdd42..0e6cf266df5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -230,7 +230,8 @@ All long-running daemons expose on `127.0.0.1`: ## Key Files to Reference - `Makefile`, `Makefile.include` — build and development targets -- `docker-compose.yml` — development environment (PMM Server, renderer, watchtower) +- `docker-compose.dev.yml` — development environment (PMM Server, renderer, watchtower) +- `docker-compose.yml` — community/quickstart compose (stable image, minimal config) - `go.mod` — Go module definition - `.golangci.yml` — linter configuration - `.mockery.yaml` — mock generation configuration diff --git a/Makefile b/Makefile index 3f1098530c2..5c74f71a174 100644 --- a/Makefile +++ b/Makefile @@ -1,48 +1,125 @@ -# Host Makefile. +# Development Makefile +.PHONY: default help init release gen test clean all +.DEFAULT_GOAL := help -include Makefile.include -include documentation/Makefile +-include .devcontainer/Makefile -ifeq ($(PROFILES),) -PROFILES := 'pmm' -endif +default: help -env-up: ## Start devcontainer +help: ## Display this help message + @echo "Please use \`make \`, where is one of the following:" + @grep -h '^[a-zA-Z]' $(MAKEFILE_LIST) | awk -F ':.*## ' 'NF==2 {printf " %-26s%s\n", $$1, $$2}' | sort + @echo + @echo For developers: check docker-compose.dev.yml to see which environment variables are available. + +init: ## Install tools + rm -rf bin/* + cd tools && go generate -x -tags=tools + + # Install golangci-lint + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./bin v2.6.2 # Version should match specified in CI + + +PROFILES ?= pmm +COMPOSE_FILE ?= docker-compose.dev.yml + +env-up: ## Start devcontainer COMPOSE_PROFILES=$(PROFILES) \ - docker compose up -d --wait --wait-timeout 100 + docker compose -f $(COMPOSE_FILE) up -d --wait --wait-timeout 100 env-up-rebuild: env-update-image ## Rebuild and start devcontainer. Useful for custom $PMM_SERVER_IMAGE COMPOSE_PROFILES=$(PROFILES) \ - docker compose up --build -d + docker compose -f $(COMPOSE_FILE) up --build -d -env-update-image: ## Pull latest dev image +env-update-image: ## Pull latest dev image COMPOSE_PROFILES=$(PROFILES) \ - docker compose pull + docker compose -f $(COMPOSE_FILE) pull -env-compose-up: env-update-image +env-compose-up: env-update-image ## Pull the image, then start devcontainer waiting for it to be ready COMPOSE_PROFILES=$(PROFILES) \ - docker compose up --detach --renew-anon-volumes --remove-orphans --wait --wait-timeout 100 + docker compose -f $(COMPOSE_FILE) up -d --renew-anon-volumes --remove-orphans --wait --wait-timeout 100 -env-devcontainer: +env-devcontainer: ## Provision devcontainer (run this after `make env-up` or `make env-compose-up`) docker exec -it --workdir=/root/go/src/github.com/percona/pmm --user root pmm-server python .devcontainer/setup.py -env-down: ## Stop devcontainer +env-down: ## Stop devcontainer COMPOSE_PROFILES=$(PROFILES) \ - docker compose down --remove-orphans + docker compose -f $(COMPOSE_FILE) down --remove-orphans -env-remove: +env-remove: ## Stop devcontainer and remove volumes COMPOSE_PROFILES=$(PROFILES) \ - docker compose down --volumes --remove-orphans + docker compose -f $(COMPOSE_FILE) down --volumes --remove-orphans TARGET ?= _bash -env: ## Run `make TARGET` in devcontainer (`make env TARGET=help`); TARGET defaults to bash - COMPOSE_PROFILES=$(PROFILES) \ +env: ## Run `make TARGET` in devcontainer (`make env TARGET=help`); TARGET defaults to bash docker exec -it --workdir=/root/go/src/github.com/percona/pmm pmm-server make $(TARGET) -env-root: ## Run `make TARGET` in devcontainer (`make env-root TARGET=help`); TARGET defaults to bash - COMPOSE_PROFILES=$(PROFILES) \ +env-root: ## Run `make TARGET` in devcontainer (`make env-root TARGET=help`); TARGET defaults to bash docker exec -it --workdir=/root/go/src/github.com/percona/pmm --user root pmm-server make $(TARGET) -rotate-encryption: ## Rotate encryption key +rotate-encryption: ## Rotate encryption key go run ./encryption-rotation/main.go + +release: ## Build release versions of all components + make -C agent release + make -C admin release + make -C managed release + make -C qan-api2 release + +gen: clean ## Generate files + make -C api gen + make -C api clean-swagger + + make -C agent gen + make -C admin gen + make -C managed gen + + make gen-mocks ## Generate mocks + + make format + make format ## TODO: One formatting run is not enough, figure out why. + go install -v ./... + +clean: ## Remove generated files + make -C api clean + +gen-mocks: ## Generate mocks for API + find . -name mock_*.go -delete + ./bin/mockery --config .mockery.yaml + +test-common: ## Run tests from API (and other shared) packages only (i.e it ignores directories that are explicitly listed) + go test $(shell go list ./... | grep -v -e admin -e agent -e managed -e api-tests -e qan-api2 -e update) + +api-test: ## Run API tests on dev env. + go test -count=1 -race -p 1 -v ./api-tests/... -pmm.server-insecure-tls + +GOLANG_CI_LINT_RUN_OPTS ?= +check: ## Run required checks and linters + bin/buf lint -v api + LOG_LEVEL=error bin/golangci-lint run -c=.golangci.yml --new-from-rev=$(shell git merge-base v3 HEAD) --new $(GOLANG_CI_LINT_RUN_OPTS) + bin/go-sumtype ./... + +check-license: ## Run license header checks against source files + bin/license-eye -c .licenserc.yaml header check + +check-all: check-license check ## Run linter and license checks + +check-new: ## Run linters only against new code since v3 branch point + bin/golangci-lint run -c=.golangci.yml --new-from-rev=origin/v3 --new + +FILES = $(shell find . -type f -name '*.go') + +format: ## Format source code + make -C api format + bin/gofumpt -l -w $(FILES) + bin/goimports -local github.com/percona/pmm -l -w $(FILES) + bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES) + +serve: ## Serve API documentation with nginx + nginx -p . -c api/nginx/nginx.conf + +GOLANG_CI_LINT_RUN_OPTS=--fix +prepare-pr: gen check-all ## Run all checks and generate files + go mod tidy diff --git a/Makefile.include b/Makefile.include deleted file mode 100644 index 9a4ef9018b9..00000000000 --- a/Makefile.include +++ /dev/null @@ -1,78 +0,0 @@ - -.PHONY: default init release gen test clean all -default: help - -help: ## Display this help message - @echo "Please use \`make \`, where is one of the following:" - @grep -h '^[a-zA-Z]' $(MAKEFILE_LIST) | awk -F ':.*## ' 'NF==2 {printf " %-26s%s\n", $$1, $$2}' - @echo - @echo Check docker-compose.yml to see which environment variables are available. - -init: ## Install tools - rm -rf bin/* - cd tools && go generate -x -tags=tools - - # Install golangci-lint - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./bin v2.6.2 # Version should match specified in CI - -release: ## Build release versions of all components - make -C agent release - make -C admin release - make -C managed release - make -C qan-api2 release - -gen: clean ## Generate files - make -C api gen - make -C api clean-swagger - - make -C agent gen - make -C admin gen - make -C managed gen - - make gen-mocks ## Generate mocks - - make format - make format ## TODO: One formatting run is not enough, figure out why. - go install -v ./... - -clean: ## Remove generated files - make -C api clean - -gen-mocks: - find . -name mock_*.go -delete - ./bin/mockery --config .mockery.yaml - -test-common: ## Run tests from API (and other shared) packages only (i.e it ignores directories that are explicitly listed) - go test $(shell go list ./... | grep -v -e admin -e agent -e managed -e api-tests -e qan-api2 -e update) - -api-test: ## Run API tests on dev env. - go test -count=1 -race -p 1 -v ./api-tests/... -pmm.server-insecure-tls - -GOLANG_CI_LINT_RUN_OPTS ?= -check: ## Run required checks and linters - bin/buf lint -v api - LOG_LEVEL=error bin/golangci-lint run -c=.golangci.yml --new-from-rev=$(shell git merge-base v3 HEAD) --new $(GOLANG_CI_LINT_RUN_OPTS) - bin/go-sumtype ./... - -check-license: ## Run license header checks against source files - bin/license-eye -c .licenserc.yaml header check - -check-all: check-license check ## Run linter and license checks - -check-new: - bin/golangci-lint run -c=.golangci.yml --new-from-rev=origin/v3 --new - -FILES = $(shell find . -type f -name '*.go') - -format: ## Format source code - make -C api format - bin/gofumpt -l -w $(FILES) - bin/goimports -local github.com/percona/pmm -l -w $(FILES) - bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES) - -serve: ## Serve API documentation with nginx - nginx -p . -c api/nginx/nginx.conf - -GOLANG_CI_LINT_RUN_OPTS=--fix -prepare-pr: gen check-all - go mod tidy diff --git a/README.md b/README.md index 3de1c8a6a52..f4f41a608b6 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ See the [PMM Documentation](https://docs.percona.com/percona-monitoring-and-mana * Spot critical performance issues faster, understand the root cause of incidents better and troubleshoot them more efficiently. * Zoom-in, drill-down database performance from node to single query levels. Perform in-depth troubleshooting and performance optimization. * Built-in Advisors run regular checks of the databases connected to PMM. The checks identify and alert you of potential security threats, performance degradation, data loss and data corruption. -* Backup and restore databases up to a specific moment with Point-in-Time-Recovery. ## Architecture @@ -41,7 +40,7 @@ See the [PMM Documentation](https://docs.percona.com/percona-monitoring-and-mana ## Installation -There are numbers of installation methods, please check our [About PMM installation](https://docs.percona.com/percona-monitoring-and-management/3/install-pmm/index.html) documentation page. +There are different installation methods, please check our [About PMM installation](https://docs.percona.com/percona-monitoring-and-management/3/install-pmm/index.html) documentation page. But in a nutshell: @@ -61,7 +60,7 @@ $ docker run --detach --restart always \ --name pmm-server \ percona/pmm-server:3 ``` -4. Start a web browser and in the address bar enter the server name or IP address of the PMM server host. +4. Start a web browser and type the server name or IP address of the PMM server host (defaults to https://localhost). PMM Login Page diff --git a/agent/cmd/pmm-agent-entrypoint/main.go b/agent/cmd/pmm-agent-entrypoint/main.go index ea878c55609..0d82e1f6004 100644 --- a/agent/cmd/pmm-agent-entrypoint/main.go +++ b/agent/cmd/pmm-agent-entrypoint/main.go @@ -40,9 +40,9 @@ It is configured entirely by environment variables. Arguments or flags are not u The following environment variables are recognized by the Docker entrypoint: * PMM_AGENT_SETUP - if true, 'pmm-agent setup' is called before 'pmm-agent run'. -* PMM_AGENT_PRERUN_FILE - if non-empty, runs given file with 'pmm-agent run' running in the background. -* PMM_AGENT_PRERUN_SCRIPT - if non-empty, runs given shell script content with 'pmm-agent run' running in the background. -* PMM_AGENT_SIDECAR - if true, 'pmm-agent' will be restarted in case of it's failed. +* PMM_AGENT_PRERUN_FILE - if non-empty, runs given file while 'pmm-agent run' is running in the background. +* PMM_AGENT_PRERUN_SCRIPT - if non-empty, runs given shell script content while 'pmm-agent run' is running in the background. +* PMM_AGENT_SIDECAR - if true, 'pmm-agent' will be restarted in case it fails. * PMM_AGENT_SIDECAR_SLEEP - time to wait before restarting pmm-agent if PMM_AGENT_SIDECAR is true. 1 second by default. Additionally, the many environment variables are recognized by pmm-agent itself. diff --git a/agent/docker-compose.yml b/agent/docker-compose.yml index 5adf0b51518..9cc92e424e8 100644 --- a/agent/docker-compose.yml +++ b/agent/docker-compose.yml @@ -11,7 +11,6 @@ services: - PMM_DEV_ADVISOR_CHECKS_FILE=/srv/checks/custom-checks.yml # for check-dev # - PMM_DEV_PERCONA_PLATFORM_ADDRESS=check-dev.percona.com:443 - # - PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY=RWTg+ZmCCjt7O8eWeAmTLAqW+1ozUbpRSKSwNTmO+exlS5KEIPYWuYdX volumes: - ./testdata/checks:/srv/checks @@ -45,7 +44,7 @@ services: - ./testdata/mysql:/mysql mongo: - image: ${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} + image: ${MONGO_IMAGE:-percona/percona-server-mongodb:6.0} container_name: pmm-agent_mongo command: - --profile=2 @@ -60,7 +59,7 @@ services: - ./testdata/mongo/var/log/mongodb:/var/log/mongodb mongo_with_tls: - image: ${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} + image: ${MONGO_IMAGE:-percona/percona-server-mongodb:6.0} container_name: pmm-agent_mongo_with_tls command: - --profile=2 @@ -78,17 +77,17 @@ services: - ./testdata/mongo/var/log/mongodb:/var/log/mongodb mongonoauth: - image: ${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} + image: ${MONGO_IMAGE:-percona/percona-server-mongodb:6.0} container_name: pmm-agent_mongonoauth command: --profile=2 -bind_ip=0.0.0.0 --noauth ports: - "127.0.0.1:27019:27017" mongo_repl: - image: ${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} + image: ${MONGO_IMAGE:-percona/percona-server-mongodb:6.0} container_name: pmm-agent_mongorepl environment: - - MONGO_IMAGE=${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} + - MONGO_IMAGE=${MONGO_IMAGE:-percona/percona-server-mongodb:6.0} volumes: - ./testscripts/:/scripts command: /scripts/mongo_repl.sh @@ -97,11 +96,11 @@ services: - "127.0.0.1:27021:27021" mongo_repl_with_tls: - image: ${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} + image: ${MONGO_IMAGE:-percona/percona-server-mongodb:6.0} container_name: pmm-agent_mongorepl_with_tls command: /scripts/mongo_repl_with_tls.sh environment: - - MONGO_IMAGE=${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} + - MONGO_IMAGE=${MONGO_IMAGE:-percona/percona-server-mongodb:6.0} ports: - "127.0.0.1:27022:27022" - "127.0.0.1:27023:27023" @@ -110,7 +109,7 @@ services: - ${PWD}/utils/tests/testdata/mongodb:/etc/tls/certificates postgres: - image: ${POSTGRES_IMAGE:-postgres:11} + image: ${POSTGRES_IMAGE:-postgres:14} container_name: pmm-agent_postgres command: > -c shared_preload_libraries='${PG_PRELOADED_LIBS:-pg_stat_statements}' diff --git a/api-tests/docker-compose.yml b/api-tests/docker-compose.yml index e75134789a5..464fdd6ae71 100644 --- a/api-tests/docker-compose.yml +++ b/api-tests/docker-compose.yml @@ -11,7 +11,6 @@ services: # - PMM_DEV_ADVISOR_CHECKS_FILE=/srv/checks/custom-checks.yml # for check-dev - PMM_DEV_PERCONA_PLATFORM_ADDRESS=check-dev.percona.com - - PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY=RWTg+ZmCCjt7O8eWeAmTLAqW+1ozUbpRSKSwNTmO+exlS5KEIPYWuYdX volumes: - ./testdata/checks:/srv/checks diff --git a/dashboards/pmm-app/docker-compose.yaml b/dashboards/pmm-app/docker-compose.yaml index 22e05157edf..fb1815b6a91 100644 --- a/dashboards/pmm-app/docker-compose.yaml +++ b/dashboards/pmm-app/docker-compose.yaml @@ -8,9 +8,7 @@ services: - './dist:/srv/grafana/plugins/pmm-app/dist' environment: - PMM_DEBUG=1 - - PMM_DEV_PORTAL_URL=https://portal-dev.percona.com - PMM_DEV_PERCONA_PLATFORM_ADDRESS=https://check-dev.percona.com:443 - - PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY=RWTkF7Snv08FCboTne4djQfN5qbrLfAjb8SY3/wwEP+X5nUrkxCEvUDJ ports: - 80:8080 - 443:8443 diff --git a/dev/init-pg.sql b/dev/init-pg.sql new file mode 100644 index 00000000000..841ff0c4b19 --- /dev/null +++ b/dev/init-pg.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; diff --git a/dev/my.cnf b/dev/my.cnf new file mode 100644 index 00000000000..6f4f4977ba2 --- /dev/null +++ b/dev/my.cnf @@ -0,0 +1,7 @@ +[mysqld] +slow_query_log = ON +slow_query_log_file = /var/log/mysql/mysql-slow.log +slow_query_log_always_write_time = 0 +long_query_time = 0 +log_slow_admin_statements = ON +log_slow_extra = ON diff --git a/dev/mysql.logrotate b/dev/mysql.logrotate new file mode 100644 index 00000000000..2dfa7190962 --- /dev/null +++ b/dev/mysql.logrotate @@ -0,0 +1,14 @@ +/var/log/mysql/mysql-slow.log { + daily + missingok + rotate 7 + compress + delaycompress + notifempty + sharedscripts + postrotate + if [ -f /var/run/mysqld/mysqld.pid ]; then + kill -HUP $(cat /var/run/mysqld/mysqld.pid) + fi + endscript +} diff --git a/dev/sysbench-ps.sh b/dev/sysbench-ps.sh new file mode 100644 index 00000000000..8f2007ee981 --- /dev/null +++ b/dev/sysbench-ps.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -o xtrace + +sysbench \ + --db-driver=mysql \ + --mysql-host=ps \ + --mysql-port=3306 \ + --mysql-user=root \ + --mysql-password=secret \ + --mysql-db=sbtest \ + --table-size=1000000 \ + oltp_read_write \ + prepare + +sysbench \ + --rate=200 \ + --threads=64 \ + --report-interval=10 \ + --time=0 \ + --events=0 \ + --rand-type=pareto \ + --db-driver=mysql \ + --mysql-host=ps \ + --mysql-port=3306 \ + --mysql-user=root \ + --mysql-password=secret \ + --mysql-db=sbtest \ + --table-size=1000000 \ + oltp_read_only \ + run diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000000..5ee64696877 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,148 @@ +--- +services: + pmm-server: + profiles: + - pmm + image: ${PMM_CONTAINER:-perconalab/pmm-server:3-dev-container} + platform: linux/amd64 + container_name: pmm-server + hostname: pmm-server + networks: + - ${NETWORK:-default} + environment: + - AWS_ACCESS_KEY=${AWS_ACCESS_KEY:-} + - AWS_SECRET_KEY=${AWS_SECRET_KEY:-} + - GF_EMAILS_WELCOME_EMAIL_ON_SIGN_UP=1 + - GF_SMTP_ENABLED=1 + - GF_SMTP_HOST=mail-server:1025 + - GF_SMTP_FROM_NAME=${GF_SMTP_FROM_NAME:-Percona} + - GF_SMTP_FROM_ADDRESS=${GF_SMTP_FROM_ADDRESS:-admin@percona.com} + - REVIEWDOG_GITHUB_API_TOKEN=${REVIEWDOG_GITHUB_API_TOKEN:-} + - PMM_RELEASE_PATH=/root/go/bin + - PMM_ENABLE_ACCESS_CONTROL=${PMM_ENABLE_ACCESS_CONTROL:-0} + - PMM_WATCHTOWER_HOST=${PMM_WATCHTOWER_HOST:-http://watchtower:8080} + - PMM_WATCHTOWER_TOKEN=${PMM_WATCHTOWER_TOKEN:-INSECURE_TOKEN} + - PMM_ENABLE_NOMAD=${PMM_ENABLE_NOMAD:-0} + - PMM_PUBLIC_ADDRESS=${PMM_PUBLIC_ADDRESS:-localhost} + - PMM_RELEASE_VERSION=3.7.1 + - GO_VERSION=1.25.x + # - PMM_ENABLE_INTERNAL_PG_QAN=${PMM_ENABLE_INTERNAL_PG_QAN:-1} + # - PMM_DISTRIBUTION_METHOD=${PMM_DISTRIBUTION_METHOD:-docker} + # - PMM_DEV_UPDATE_DOCKER_IMAGE=${PMM_DEV_UPDATE_DOCKER_IMAGE:-} + # - PMM_DEV_PERCONA_PLATFORM_ADDRESS=https://check.localhost + # - PMM_DEV_PERCONA_PLATFORM_INSECURE=1 + # - PMM_DEV_TELEMETRY_INTERVAL=10s + # - PMM_DEV_TELEMETRY_DISABLE_START_DELAY=1 + # - PMM_DEV_TELEMETRY_RETRY_BACKOFF=10s + # - PMM_CLICKHOUSE_ADDR=127.0.0.1:9000 + # - PMM_CLICKHOUSE_DATABASE=pmm + # - PMM_CLICKHOUSE_USER=default + # - PMM_CLICKHOUSE_PASSWORD= + # - PMM_CLICKHOUSE_IS_CLUSTER=1 + # - PMM_CLICKHOUSE_CLUSTER_NAME=pmmclickhouse + # - PMM_DEBUG=1 + # - PMM_DEV_ADVISOR_CHECKS_FILE=/srv/checks/local-checks.yml + # - PMM_POSTGRES_ADDR=pg + # - PMM_POSTGRES_DBNAME=pmm-managed + # - PMM_POSTGRES_USERNAME=pmm-managed + # - PMM_POSTGRES_DBPASSWORD= + # - PMM_POSTGRES_SSL_MODE=verify-full + # - PMM_POSTGRES_SSL_CA_PATH=/tmp/certs/root.crt + # - PMM_POSTGRES_SSL_KEY_PATH=/tmp/certs/pmm-managed.key + # - PMM_POSTGRES_SSL_CERT_PATH=/tmp/certs/pmm-managed.crt + # - PMM_DISABLE_BUILTIN_POSTGRES=1 + - GF_RENDERING_SERVER_URL=http://renderer:8081/render + - GF_RENDERING_CALLBACK_URL=https://pmm-server:8443/graph/ + # - GF_DATABASE_SSL_MODE=verify-full + # - GF_DATABASE_CA_CERT_PATH=/tmp/certs/root.crt + # - GF_DATABASE_CLIENT_KEY_PATH=/tmp/certs/grafana.key + # - GF_DATABASE_CLIENT_CERT_PATH=/tmp/certs/grafana.crt + + # for delve + cap_add: + - SYS_PTRACE + security_opt: + - seccomp:unconfined + + # see https://github.com/golang/go/wiki/LinuxKernelSignalVectorBug#what-to-do + ulimits: + memlock: 67108864 + + ports: + - ${PMM_PORT_HTTPS:-443}:8443 + # For headless delve + - ${PMM_PORT_DELVE:-2345}:2345 + # PG + - ${PMM_PORT_PG:-5432}:5432 + # VM + - ${PMM_PORT_VM:-9090}:9090 + # CH + - ${PMM_PORT_CH_TCP:-19000}:9000 + - ${PMM_PORT_CH_HTTP:-8123}:8123 + volumes: + - ./:/root/go/src/github.com/percona/pmm + - ./managed/testdata/updater/:/home/pmm/.config/systemd/user/ + # Dev clickhouse config for backup/restore + - ./dev/clickhouse-config.xml:/etc/clickhouse-server/config.xml:ro + # caching + - go-modules:/root/go/pkg/mod + - go-cache:/root/.cache + - pmm-data:/srv + # mount advisor files for easy development + - ./managed/data/advisors/:/usr/local/percona/advisors/ + - ./managed/data/checks/:/usr/local/percona/checks/ + - ./managed/data/alerting-templates/:/usr/local/percona/alerting-templates/ + # grafana + # - "../grafana:/workspace" + # - "../grafana/public:/usr/share/grafana/public" + # - "../grafana/conf/grafana.local-dev.ini:/usr/share/grafana/conf/defaults.ini" + # command: > + # bash -c " + # # script you want to run before entrypoint, e.g. + # bash /tmp/wait-for-it.sh -t 30 some-service:some-port -- + # /opt/entrypoint.sh + # " + + mail-server: + profiles: + - pmm + image: axllent/mailpit + container_name: mail-server + hostname: mail-server + ports: + - 8025:8025 # Web UI - http://localhost:8025 + networks: + - ${PMM_NETWORK:-default} + + renderer: + image: grafana/grafana-image-renderer:latest + environment: + - RENDERING_VERBOSE_LOGGING=true + - RENDERING_DUMPIO=true + - IGNORE_HTTPS_ERRORS=true + - LOG_LEVEL=debug + networks: + - ${NETWORK:-default} + ports: + - 8081:8081 + + watchtower: + profiles: + - pmm + image: ${WATCHTOWER_IMAGE:-perconalab/watchtower} + container_name: watchtower + environment: + - WATCHTOWER_HTTP_LISTEN_PORT=8080 + - WATCHTOWER_HTTP_API_UPDATE=1 + - WATCHTOWER_HTTP_API_TOKEN=${PMM_WATCHTOWER_TOKEN:-INSECURE_TOKEN} + - WATCHTOWER_NO_RESTART=${WATCHTOWER_NO_RESTART:-0} + hostname: watchtower + networks: + - ${NETWORK:-default} + volumes: + - ${DOCKER_PATH:-/var/run/docker.sock}:/var/run/docker.sock + +volumes: + go-modules: + go-cache: + pmm-data: diff --git a/docker-compose.yml b/docker-compose.yml index 5af8ee99faf..5f9084197a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,133 +1,155 @@ --- +# Usage: +# minimal suite (server, client, mysql + sysbench): +# docker compose --profile pmm up -d +# with QAN overrides (dev image + external ClickHouse): +# docker compose -f docker-compose.yml -f qan-api2/docker-compose.yml --profile pmm up -d +# full suite (server, client, postgres, mongo, mysql + sysbench, external clickhouse): +# docker compose -f docker-compose.yml -f qan-api2/docker-compose.yml --profile pmm --profile postgres --profile mongo up -d +# Note: database ports are initially disabled since the communication takes place on a shared bridge network. services: pmm-server: profiles: - pmm - image: ${PMM_CONTAINER:-perconalab/pmm-server:3-dev-container} + image: ${PMM_SERVER_IMAGE:-percona/pmm-server:3} platform: linux/amd64 container_name: pmm-server hostname: pmm-server - networks: - - ${NETWORK:-default} + restart: always environment: - - AWS_ACCESS_KEY=${AWS_ACCESS_KEY:-} - - AWS_SECRET_KEY=${AWS_SECRET_KEY:-} - - REVIEWDOG_GITHUB_API_TOKEN=${REVIEWDOG_GITHUB_API_TOKEN:-} - - PMM_RELEASE_PATH=/root/go/bin - - PMM_ENABLE_ACCESS_CONTROL=${PMM_ENABLE_ACCESS_CONTROL:-0} - - PMM_WATCHTOWER_HOST=${PMM_WATCHTOWER_HOST:-http://watchtower:8080} - - PMM_WATCHTOWER_TOKEN=${PMM_WATCHTOWER_TOKEN:-INSECURE_TOKEN} - - PMM_ENABLE_NOMAD=${PMM_ENABLE_NOMAD:-0} + - GF_SECURITY_ADMIN_USER=${PMM_ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${PMM_ADMIN_PASSWORD:-admin} + - PMM_ENABLE_ACCESS_CONTROL=${PMM_ENABLE_ACCESS_CONTROL:-1} - PMM_PUBLIC_ADDRESS=${PMM_PUBLIC_ADDRESS:-localhost} - - PMM_RELEASE_VERSION=3.6.0 - - GO_VERSION=1.25.x - # - PMM_ENABLE_INTERNAL_PG_QAN=${PMM_ENABLE_INTERNAL_PG_QAN:-1} - # - PMM_DISTRIBUTION_METHOD=${PMM_DISTRIBUTION_METHOD:-docker} - # - PMM_DEV_UPDATE_DOCKER_IMAGE=${PMM_DEV_UPDATE_DOCKER_IMAGE:-} - # - PMM_DEV_PERCONA_PLATFORM_ADDRESS=https://check.localhost - # - PMM_DEV_PERCONA_PLATFORM_INSECURE=1 - # - PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY= - # - PMM_DEV_TELEMETRY_INTERVAL=10s - # - PMM_DEV_TELEMETRY_DISABLE_START_DELAY=1 - # - PMM_DEV_TELEMETRY_RETRY_BACKOFF=10s + - PMM_ENABLE_INTERNAL_PG_QAN=${PMM_ENABLE_INTERNAL_PG_QAN:-0} + - PMM_DEBUG=${PMM_DEBUG:-0} + # To enable an external ClickHouse database # - PMM_CLICKHOUSE_ADDR=127.0.0.1:9000 # - PMM_CLICKHOUSE_DATABASE=pmm # - PMM_CLICKHOUSE_USER=default # - PMM_CLICKHOUSE_PASSWORD= - # - PMM_CLICKHOUSE_IS_CLUSTER=1 - # - PMM_CLICKHOUSE_CLUSTER_NAME=pmmclickhouse - # - PMM_DEBUG=1 - # - PMM_DEV_ADVISOR_CHECKS_FILE=/srv/checks/local-checks.yml + # - PMM_DISABLE_BUILTIN_CLICKHOUSE=1 + # To enable an external PostgreSQL database # - PMM_POSTGRES_ADDR=pg # - PMM_POSTGRES_DBNAME=pmm-managed # - PMM_POSTGRES_USERNAME=pmm-managed # - PMM_POSTGRES_DBPASSWORD= - # - PMM_POSTGRES_SSL_MODE=verify-full - # - PMM_POSTGRES_SSL_CA_PATH=/tmp/certs/root.crt - # - PMM_POSTGRES_SSL_KEY_PATH=/tmp/certs/pmm-managed.key - # - PMM_POSTGRES_SSL_CERT_PATH=/tmp/certs/pmm-managed.crt # - PMM_DISABLE_BUILTIN_POSTGRES=1 - - GF_RENDERING_SERVER_URL=http://renderer:8081/render - - GF_RENDERING_CALLBACK_URL=https://pmm-server:8443/graph/ - # - GF_DATABASE_SSL_MODE=verify-full - # - GF_DATABASE_CA_CERT_PATH=/tmp/certs/root.crt - # - GF_DATABASE_CLIENT_KEY_PATH=/tmp/certs/grafana.key - # - GF_DATABASE_CLIENT_CERT_PATH=/tmp/certs/grafana.crt - - # for delve - cap_add: - - SYS_PTRACE - security_opt: - - seccomp:unconfined - - # see https://github.com/golang/go/wiki/LinuxKernelSignalVectorBug#what-to-do - ulimits: - memlock: 67108864 - + # To be able to discover RDS databases via UI + - AWS_ACCESS_KEY=${AWS_ACCESS_KEY:-} + - AWS_SECRET_KEY=${AWS_SECRET_KEY:-} ports: - ${PMM_PORT_HTTPS:-443}:8443 - # For headless delve - - ${PMM_PORT_DELVE:-2345}:2345 - # PG - - ${PMM_PORT_PG:-5432}:5432 - # VM - - ${PMM_PORT_VM:-9090}:9090 - # CH - - ${PMM_PORT_CH_TCP:-19000}:9000 - - ${PMM_PORT_CH_HTTP:-8123}:8123 volumes: - - ./:/root/go/src/github.com/percona/pmm - - ./Makefile.devcontainer:/root/go/src/github.com/percona/pmm/Makefile:ro # substitute Makefile in devcontainer - - ./managed/testdata/updater/:/home/pmm/.config/systemd/user/ - - ./dev/clickhouse-config.xml:/etc/clickhouse-server/config.xml:ro # Dev clickhouse config for backups/restore - # caching - - go-modules:/root/go/pkg/mod - - go-cache:/root/.cache - - srv:/srv - # mount advisor files for easy development - - ./managed/data/advisors/:/usr/local/percona/advisors/ - - ./managed/data/checks/:/usr/local/percona/checks/ - - ./managed/data/alerting-templates/:/usr/local/percona/alerting-templates/ - # grafana - # - "../grafana:/workspace" - # - "../grafana/public:/usr/share/grafana/public" - # - "../grafana/conf/grafana.local-dev.ini:/usr/share/grafana/conf/defaults.ini" - # command: > - # bash -c " - # # script you want to run before entrypoint, e.g. - # bash /tmp/wait-for-it.sh -t 30 some-service:some-port -- - # /opt/entrypoint.sh - # " + - pmm-data:/srv - renderer: - image: grafana/grafana-image-renderer:latest + pmm-client: + profiles: + - pmm + image: percona/pmm-client:3 + container_name: pmm-client + depends_on: + pmm-server: + condition: service_healthy + sysbench-ps: + condition: service_started + postgres: + condition: service_started + required: false + mongo: + condition: service_started + required: false environment: - - RENDERING_VERBOSE_LOGGING=true - - RENDERING_DUMPIO=true - - IGNORE_HTTPS_ERRORS=true - - LOG_LEVEL=debug - networks: - - ${NETWORK:-default} - ports: - - 8081:8081 + PMM_AGENT_SERVER_ADDRESS: pmm-server:8443 + PMM_AGENT_SERVER_USERNAME: admin + PMM_AGENT_SERVER_PASSWORD: admin + PMM_AGENT_SERVER_INSECURE_TLS: 1 + PMM_AGENT_CONFIG_FILE: config/pmm-agent.yaml + PMM_AGENT_SETUP: 1 + PMM_AGENT_SETUP_FORCE: 1 + PMM_AGENT_SIDECAR: 1 + PMM_AGENT_PRERUN_SCRIPT: | + pmm-admin status --wait=10s + pmm-admin add mysql --cluster=dev-cluster --environment=dev --custom-labels='app=test-app' --username=root --password=secret --query-source=perfschema mysql-dev ps:3306 + pmm-admin add postgresql --username=postgres --password=secret postgres-dev postgres:5432 || true + pmm-admin add mongodb --username=mongo --password=secret mongo-dev mongo:27017 || true - watchtower: + ps: profiles: - pmm - image: ${WATCHTOWER_IMAGE:-perconalab/watchtower} - container_name: watchtower + image: percona/percona-server:latest + container_name: ps environment: - - WATCHTOWER_HTTP_LISTEN_PORT=8080 - - WATCHTOWER_HTTP_API_UPDATE=1 - - WATCHTOWER_HTTP_API_TOKEN=${PMM_WATCHTOWER_TOKEN:-INSECURE_TOKEN} - - WATCHTOWER_NO_RESTART=${WATCHTOWER_NO_RESTART:-0} - hostname: watchtower - networks: - - ${NETWORK:-default} + - MYSQL_ROOT_PASSWORD=secret + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-P", "3306", "-u", "root", "-psecret"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + volumes: + - ./dev/my.cnf:/etc/my.cnf + - ./dev/mysql.logrotate:/etc/logrotate.d/mysql + # ports: + # - 3306:3306 + + ps-init: + profiles: + - pmm + image: percona/percona-server:latest + container_name: ps-init + depends_on: + ps: + condition: service_healthy + command: mysql -h ps -u root -psecret -e "CREATE DATABASE IF NOT EXISTS sbtest;" + + postgres: + profiles: + - postgres + image: postgres:18 + container_name: postgres + command: > + -c shared_preload_libraries=pg_stat_statements + -c track_activity_query_size=2048 + -c pg_stat_statements.track=all + -c pg_stat_statements.max=10000 + -c pg_stat_statements.track=all + -c pg_stat_statements.save=off + -c track_io_timing=on + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=secret + volumes: + - ./dev/init-pg.sql:/docker-entrypoint-initdb.d/init-pg.sql + # ports: + # - 5432:5432 + + mongo: + profiles: + - mongo + image: mongo:8-noble + container_name: mongo + environment: + - MONGO_INITDB_ROOT_USERNAME=mongo + - MONGO_INITDB_ROOT_PASSWORD=secret + # ports: + # - 27017:27017 + + sysbench-ps: + profiles: + - pmm + image: perconalab/sysbench + platform: linux/amd64 + container_name: sysbench-ps + depends_on: + ps-init: + condition: service_completed_successfully + pmm-server: + condition: service_healthy + command: bash /sysbench-ps.sh volumes: - - ${DOCKER_PATH:-/var/run/docker.sock}:/var/run/docker.sock + - ./dev/sysbench-ps.sh:/sysbench-ps.sh volumes: - go-modules: - go-cache: - srv: + pmm-data: + name: pmm-data diff --git a/docs/process/v2_to_v3_environment_variables.md b/docs/process/v2_to_v3_environment_variables.md index f95ad9ff4e4..6d255d60ab8 100644 --- a/docs/process/v2_to_v3_environment_variables.md +++ b/docs/process/v2_to_v3_environment_variables.md @@ -17,8 +17,8 @@ Below is a list of affected variables and their new names. | `METRICS_RESOLUTION_HR` | `PMM_METRICS_RESOLUTION_HR` | | | `METRICS_RESOLUTION_LR` | `PMM_METRICS_RESOLUTION_LR` | | | `METRICS_RESOLUTION_MR` | `PMM_METRICS_RESOLUTION_MR` | | -| `OAUTH_PMM_CLIENT_ID` | `PMM_DEV_OAUTH_CLIENT_ID` | | -| `OAUTH_PMM_CLIENT_SECRET` | `PMM_DEV_OAUTH_CLIENT_SECRET` | | +| `OAUTH_PMM_CLIENT_ID` | `PMM_DEV_OAUTH_CLIENT_ID` | Removed in PMM v3 | +| `OAUTH_PMM_CLIENT_SECRET` | `PMM_DEV_OAUTH_CLIENT_SECRET` | Removed in PMM v3 | | `PERCONA_TEST_AUTH_HOST` | | Removed in PMM v3, use `PMM_DEV_PERCONA_PLATFORM_ADDRESS` | | `PERCONA_TEST_CHECKS_FILE` | `PMM_DEV_ADVISOR_CHECKS_FILE` | | | `PERCONA_TEST_CHECKS_HOST` | | Removed in PMM v3, use `PMM_DEV_PERCONA_PLATFORM_ADDRESS` | diff --git a/documentation/Makefile b/documentation/Makefile index e86de908724..910b086004a 100644 --- a/documentation/Makefile +++ b/documentation/Makefile @@ -1,12 +1,6 @@ # Documentation Makefile. -DOCDIR := documentation - -ifeq ($(shell basename $(CURDIR)),$(DOCDIR)) - DIR := $(CURDIR) -else - DIR := $(CURDIR)/$(DOCDIR) -endif +DIR := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST))))) doc-check-images: ## Check if all images are used in documentation @$(DIR)/resources/bin/check-images.sh diff --git a/documentation/docs/reference/index.md b/documentation/docs/reference/index.md index 93e04282e18..3a47cccd8e1 100644 --- a/documentation/docs/reference/index.md +++ b/documentation/docs/reference/index.md @@ -50,7 +50,7 @@ PMM Server includes the following tools: - [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) is a scalable time-series database. - [ClickHouse](https://clickhouse.com) is a third-party column-oriented database that facilitates the Query Analytics functionality. - [Grafana](http://docs.grafana.org) is a third-party dashboard and graph engine for visualizing data aggregated in an intuitive web interface. - - [PMM Dashboards](https://github.com/percona/pmm/dashboards) is a set of metrics dashboards developed by Percona. + - [PMM Dashboards](https://github.com/percona/pmm/tree/v3/dashboards) is a set of metrics dashboards developed by Percona. ### PMM Client diff --git a/documentation/docs/reference/third-party/clickhouse.md b/documentation/docs/reference/third-party/clickhouse.md index 0f49f49116f..a65196fd6b9 100644 --- a/documentation/docs/reference/third-party/clickhouse.md +++ b/documentation/docs/reference/third-party/clickhouse.md @@ -37,21 +37,21 @@ To use ClickHouse as an external database instance, provide the following enviro To use ClickHouse as an external database instance, run PMM in docker or podman with the specified variables for external ClickHouse: ```sh --e PMM_CLICKHOUSE_ADDR=$CH_HOST:$CH_PORT --e PMM_CLICKHOUSE_DATABASE=$CH_DATABASE --e PMM_CLICKHOUSE_USER=$CH_USER --e PMM_CLICKHOUSE_PASSWORD=$CH_PASSWORD +-e PMM_CLICKHOUSE_ADDR=: +-e PMM_CLICKHOUSE_DATABASE= +-e PMM_CLICKHOUSE_USER= +-e PMM_CLICKHOUSE_PASSWORD= -e PMM_DISABLE_BUILTIN_CLICKHOUSE=1 ``` Alternatively, you can use the `PMM_CLICKHOUSE_HOST` and `PMM_CLICKHOUSE_PORT` variables instead of `PMM_CLICKHOUSE_ADDR`. ```sh --e PMM_CLICKHOUSE_HOST=$CH_HOST --e PMM_CLICKHOUSE_PORT=$CH_PORT --e PMM_CLICKHOUSE_DATABASE=$CH_DATABASE --e PMM_CLICKHOUSE_USER=$CH_USER --e PMM_CLICKHOUSE_PASSWORD=$CH_PASSWORD +-e PMM_CLICKHOUSE_HOST= +-e PMM_CLICKHOUSE_PORT= +-e PMM_CLICKHOUSE_DATABASE= +-e PMM_CLICKHOUSE_USER= +-e PMM_CLICKHOUSE_PASSWORD= -e PMM_DISABLE_BUILTIN_CLICKHOUSE=1 ``` diff --git a/get-pmm.sh b/get-pmm.sh index 62c0060fc87..992acebbaf0 100755 --- a/get-pmm.sh +++ b/get-pmm.sh @@ -338,10 +338,8 @@ ENV_MAPPING=( "PERCONA_TEST_AUTH_HOST=PMM_DEV_PERCONA_PLATFORM_ADDRESS" "PERCONA_TEST_CHECKS_FILE=PMM_DEV_ADVISOR_CHECKS_FILE" "PERCONA_TEST_CHECKS_HOST=PMM_DEV_PERCONA_PLATFORM_ADDRESS" - "PERCONA_TEST_CHECKS_PUBLIC_KEY=PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY" "PERCONA_TEST_PLATFORM_ADDRESS=PMM_DEV_PERCONA_PLATFORM_ADDRESS" "PERCONA_TEST_PLATFORM_INSECURE=PMM_DEV_PERCONA_PLATFORM_INSECURE" - "PERCONA_TEST_PLATFORM_PUBLIC_KEY=PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY" "PERCONA_TEST_SAAS_HOST=PMM_DEV_PERCONA_PLATFORM_ADDRESS" "PERCONA_TEST_POSTGRES_ADDR=PMM_POSTGRES_ADDR" "PERCONA_TEST_POSTGRES_DBNAME=PMM_POSTGRES_DBNAME" @@ -380,6 +378,8 @@ ENV_TO_DROP=( "PERCONA_TEST_NICER_API" "PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE" "PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE" +"PMM_DEV_OAUTH_CLIENT_ID" +"PMM_DEV_OAUTH_CLIENT_SECRET" ) # Function to get the new key from the mapping diff --git a/managed/CONTRIBUTING.md b/managed/CONTRIBUTING.md index 108a66d0920..0938a0c5d6d 100644 --- a/managed/CONTRIBUTING.md +++ b/managed/CONTRIBUTING.md @@ -73,9 +73,9 @@ The `make env-up` command starts PMM Server but doesn't configure any database i 1. Clone the pmm-admin [repo](https://github.com/percona/pmm-admin/) and install it by running `make install`. 2. Clone the pmm-agent [repo](https://github.com/percona/pmm-agent). 3. Run database instances to be monitored. You can either run your own or use the [`docker-compose.yml`](https://github.com/percona/pmm-agent/blob/master/docker-compose.yml) file provided by pmm-agent to run MySQL, PostgreSQL, and MongoDB containers using `make env-up` in the pmm-agent repo (make sure to comment out the `pmm-server` service in the docker-compose file since we are already running pmm-managed in devcontainer). -4. Open another shell session and `cd` into the pmm-agent repo, run `make setup-dev` and `make run` to set up and run pmm-agent and connect it to pmm-managed +4. Open another shell session and `cd` into the pmm-agent repo, run `make setup-dev` and `make run-all` to set up and run pmm-agent and connect it to pmm-managed 5. In another shell, use pmm-admin to add agents to the database instances and start monitoring them using `pmm-admin add mysql --username=root --password=root-password`, `pmm-admin add postgresql --username=pmm-agent --password=pmm-agent-password`, and `pmm-admin add mongodb --username=root --password=root-password`. -6. Once pmm-managed has started monitoring the databases. Log in to the web client in your browser to verify. The number of monitored instances will have increased. +6. Once pmm-managed has started monitoring the databases, log in to the web client in your browser to verify. The number of monitored instances will have increased. ## Working with Advisors @@ -84,7 +84,7 @@ Advisors are automated checks in PMM that analyze monitored environments and pro To get started: 1. Set up the devcontainer using `make env-up`. -2. Enter the container with `make env`, then run your changes with `make run`. +2. Enter the container with `make env`, then run your changes with `make run-all`. 3. [Add instances for monitoring](#add-instances-for-monitoring) so Advisors have databases to check. 4. Verify results in the PMM dashboard. Any failed Advisor checks will appear there. 5. [Develop or update Advisors](https://docs.percona.com/percona-monitoring-and-management/3/advisors/develop-advisor-checks.html) as needed. @@ -130,7 +130,7 @@ Alert Templates are located in the `data/templates` folder. If you want to contr # Internals -There are three makefiles: `Makefile` (host), `Makefile.devcontainer`, and `Makefile.include`. `Makefile.devcontainer` is mounted on top of `Makefile` inside the devcontainer (see `docker-compose.yml`) to enable `make env TARGET=target-name` usage. +There are two makefiles: `Makefile` (host/root) and `.devcontainer/Makefile` (container targets). The root Makefile includes `.devcontainer/Makefile`, so all targets are available in one place. Container-specific targets (`run-managed`, `run-agent`, `psql-managed`, etc.) are only meaningful when running inside the devcontainer via `make env TARGET=target-name`. Devcontainer initialization code is located in `.devcontainer/setup.py`. It provisions several binaries required for code development. diff --git a/qan-api2/CONTRIBUTING.md b/qan-api2/CONTRIBUTING.md index ace5bd5a524..c6660fdb8e8 100644 --- a/qan-api2/CONTRIBUTING.md +++ b/qan-api2/CONTRIBUTING.md @@ -18,7 +18,3 @@ where PMM_CONTAINER is a name of PMM-Server container. ## Testing Run `make test-env-up` to set-up environment for tests Run `make test` to run tests. - -## Vendoring - -We use [dep](https://github.com/golang/dep) to vendor dependencies. diff --git a/qan-api2/Makefile b/qan-api2/Makefile index 6a1c9104ea2..4f1886c61e8 100644 --- a/qan-api2/Makefile +++ b/qan-api2/Makefile @@ -1,6 +1,3 @@ -# PMM Demo benchmark -include Makefile.clickhouse - default: help help: ## Display this help message @@ -42,7 +39,7 @@ test-env-up: ## Start docker containers used for testing -e CLICKHOUSE_USER=default \ -e CLICKHOUSE_PASSWORD=clickhouse \ clickhouse/clickhouse-server:25.3.6.56 - make test-env + $(MAKE) test-env test-env: ## Create pmm_test DB and load test data sleep 10 @@ -81,7 +78,6 @@ _run: env-up: ## Run ClickHouse, MySQL Server and sysbench containers. Create pmm DB in ClickHouse mkdir -p logs docker compose up -d $(DCFLAGS) --force-recreate --renew-anon-volumes --remove-orphans ch sysbench-ps - # docker compose up $(DCFLAGS) ch sysbench-pstpcc sleep 60 docker exec ch-server clickhouse client -h 127.0.0.1 --password=clickhouse --query="CREATE DATABASE IF NOT EXISTS pmm;" @@ -107,3 +103,122 @@ deploy: docker exec $(PMM_CONTAINER) supervisorctl start qan-api2 docker exec $(PMM_CONTAINER) supervisorctl status docker exec $(PMM_CONTAINER) supervisorctl tail -f qan-api2 + +## ClickHouse bernchmarking +## +## NOTE: All commands down below are intended to be run from the host system (outside of any container). +## Do not run these targets inside a Docker or PMM container. +## +## Usage: +## 1. Download the PMM Demo backup +## Run: +## make download-clickhouse-backup +## +## The default backup is 2026-01-20. +## All available backups can be found here: +## https://github.com/Percona-Lab/pmm-demo-dump/releases/tag/pmm-demo +## +## To download a different backup, set the Makefile variable BACKUP_NAME +## +## 2. Restore the PMM Demo backup +## After downloading, the backup will be available in "dev/clickhouse-backups/" +## +## Example: +## dev/clickhouse-backups/20260120 +## +## Restore the backup by running: +## make restore-clickhouse-backup +## +## You must specify the latest migration applied in the backup. +## For the default backup (20260120), this is migration 21 (default). +## +## To restore a different backup, set the Makefile variable BACKUP_NAME and BACKUP_LAST_MIGRATION +## For other backups, the latest applied migration can be found +## in the release description: +## https://github.com/Percona-Lab/pmm-demo-dump/releases/tag/pmm-demo +## +## 3. Run benchmarks +## To run benchmarks for all main endpoints: +## make bench +## +## To run benchmarks for specific endpoints: +## make bench-filters +## make bench-report +## make bench-metrics +## make bench-example +## +## Each benchmark runs 10 iterations and reports AVG, MIN, and MAX timings. +## +## This allows you to compare performance before and after changes +## and ensure there is no negative performance impact. +## +## 4. Summary +## +## Run: +## make download-clickhouse-backup +## make restore-clickhouse-backup +## make bench + +# Since the ClickHouse backup can come from a different migration state (e.g., created when the latest migration was 21), +# and the current latest migration is 22, we need to ensure that the remaining migrations are applied +# to make the system fully functional after restore. +# +# We also drop the database because, in the case of mismatched migrations, it would not be possible +# to apply them correctly. Therefore, we need to provide the backup name and the last migration +# applied to the backup. + +# Default backup name and backup last migration. +BACKUP_NAME ?= 20260120 +BACKUP_LAST_MIGRATION ?= 21 + +# Download the ClickHouse backup, unzip it and prepare the directory to restore. +download-clickhouse-backup: + ./download_clickhouse_backup.sh $(BACKUP_NAME) + +# Default ClickHouse credentials. +CLICKHOUSE_USER ?= default +CLICKHOUSE_PASSWORD ?= clickhouse + +# Restore the ClickHouse backup into the pmm-server container and apply differential migrations if needed. +restore-clickhouse-backup: + docker exec pmm-server clickhouse-client --user=$(CLICKHOUSE_USER) --password=$(CLICKHOUSE_PASSWORD) --query="DROP DATABASE IF EXISTS pmm"; + docker exec pmm-server clickhouse-client --user=$(CLICKHOUSE_USER) --password=$(CLICKHOUSE_PASSWORD) --query="RESTORE DATABASE pmm FROM Disk('backup', '$(BACKUP_NAME)')"; + docker exec -u root pmm-server go run /root/go/src/github.com/percona/pmm/qan-api2/clickhouse_migrate/main.go --last-migration $(BACKUP_LAST_MIGRATION) --user CLICKHOUSE_USER=$(CLICKHOUSE_USER) --password CLICKHOUSE_PASSWORD=$(CLICKHOUSE_PASSWORD) + +# +# Default time periods for benchmarking. +# NOTE: These defaults are aligned with the default demo backup (BACKUP_NAME=20260120). +# PMM_DEMO_BENCH_PERIOD_TO may be in the future relative to the current date and is +# intended for benchmarking with demo data that includes timestamps up to 2026-01-21. +# When using a different backup or real data, override these values so they match +# the actual time range of your data to avoid empty benchmark results. +PMM_DEMO_BENCH_PERIOD_FROM ?= 2025-12-27T00:00:00+01:00 +PMM_DEMO_BENCH_PERIOD_TO ?= 2026-01-21T23:59:59+01:00 +# + +# Run bench for all main endpoints. +bench: bench-filters bench-report bench-metrics bench-example + +# Run bench for getFilters. +bench-filters: + @echo "Running Filters benchmark" + PMM_DEMO_BENCH_PERIOD_FROM="$(PMM_DEMO_BENCH_PERIOD_FROM)" PMM_DEMO_BENCH_PERIOD_TO="$(PMM_DEMO_BENCH_PERIOD_TO)" \ + go test -bench ^BenchmarkGetFilters$$ -run=^$$ -v -benchtime=10x pmm_demo_benchmark_test.go + +# Run bench for getReport. +bench-report: + @echo "Running Report benchmark" + PMM_DEMO_BENCH_PERIOD_FROM="$(PMM_DEMO_BENCH_PERIOD_FROM)" PMM_DEMO_BENCH_PERIOD_TO="$(PMM_DEMO_BENCH_PERIOD_TO)" \ + go test -bench ^BenchmarkGetReport$$ -run=^$$ -v -benchtime=10x pmm_demo_benchmark_test.go + +# Run bench for getMetrics. +bench-metrics: + @echo "Running Metrics benchmark" + PMM_DEMO_BENCH_PERIOD_FROM="$(PMM_DEMO_BENCH_PERIOD_FROM)" PMM_DEMO_BENCH_PERIOD_TO="$(PMM_DEMO_BENCH_PERIOD_TO)" \ + go test -bench ^BenchmarkGetMetrics$$ -run=^$$ -v -benchtime=10x pmm_demo_benchmark_test.go + +# Run bench for getExample. +bench-example: + @echo "Running Example benchmark" + PMM_DEMO_BENCH_PERIOD_FROM="$(PMM_DEMO_BENCH_PERIOD_FROM)" PMM_DEMO_BENCH_PERIOD_TO="$(PMM_DEMO_BENCH_PERIOD_TO)" \ + go test -bench ^BenchmarkGetExample$$ -run=^$$ -v -benchtime=10x pmm_demo_benchmark_test.go diff --git a/qan-api2/Makefile.clickhouse b/qan-api2/Makefile.clickhouse deleted file mode 100644 index a3f1f16be82..00000000000 --- a/qan-api2/Makefile.clickhouse +++ /dev/null @@ -1,111 +0,0 @@ -## -## NOTE: All commands in this Makefile are intended to be run from the host system (outside of any container). -## Do not run these targets inside a Docker or PMM container. -## -## Usage: -## 1. Download the PMM Demo backup -## Run: -## make download-clickhouse-backup -## -## The default backup is 2026-01-20. -## All available backups can be found here: -## https://github.com/Percona-Lab/pmm-demo-dump/releases/tag/pmm-demo -## -## To download a different backup, set the Makefile variable BACKUP_NAME -## -## 2. Restore the PMM Demo backup -## After downloading, the backup will be available in "dev/clickhouse-backups/" -## -## Example: -## dev/clickhouse-backups/20260120 -## -## Restore the backup by running: -## make restore-clickhouse-backup -## -## You must specify the latest migration applied in the backup. -## For the default backup (20260120), this is migration 21 (default). -## -## To restore a different backup, set the Makefile variable BACKUP_NAME and BACKUP_LAST_MIGRATION -## For other backups, the latest applied migration can be found -## in the release description: -## https://github.com/Percona-Lab/pmm-demo-dump/releases/tag/pmm-demo -## -## 3. Run benchmarks -## To run benchmarks for all main endpoints: -## make bench -## -## To run benchmarks for specific endpoints: -## make bench-filters -## make bench-report -## make bench-metrics -## make bench-example -## -## Each benchmark runs 10 iterations and reports AVG, MIN, and MAX timings. -## -## This allows you to compare performance before and after changes -## and ensure there is no negative performance impact. -## -## 4. Summary -## -## Run: -## make download-clickhouse-backup -## make restore-clickhouse-backup -## make bench -.PHONY: download-clickhouse-backup restore-clickhouse-backup bench bench-filters bench-report bench-metrics bench-example -# Since the ClickHouse backup can come from a different migration state (e.g., created when the latest migration was 21), -# and the current latest migration is 22, we need to ensure that the remaining migrations are applied -# to make the system fully functional after restore. -# -# We also drop the database because, in the case of mismatched migrations, it would not be possible -# to apply them correctly. Therefore, we need to provide the backup name and the last migration -# applied to the backup. -# -# Default backup name and backup last migration. -BACKUP_NAME ?= 20260120 -BACKUP_LAST_MIGRATION ?= 21 -# -# Download the ClickHouse backup, unzip it and prepare the directory to restore. -download-clickhouse-backup: - ./download_clickhouse_backup.sh $(BACKUP_NAME) -# -# Default ClickHouse credentials. -CLICKHOUSE_USER ?= default -CLICKHOUSE_PASSWORD ?= clickhouse -# -# Restore the ClickHouse backup into the pmm-server container and apply differential migrations if needed. -restore-clickhouse-backup: - docker exec pmm-server clickhouse-client --user=$(CLICKHOUSE_USER) --password=$(CLICKHOUSE_PASSWORD) --query="DROP DATABASE IF EXISTS pmm"; - docker exec pmm-server clickhouse-client --user=$(CLICKHOUSE_USER) --password=$(CLICKHOUSE_PASSWORD) --query="RESTORE DATABASE pmm FROM Disk('backup', '$(BACKUP_NAME)')"; - docker exec -u root pmm-server go run /root/go/src/github.com/percona/pmm/qan-api2/clickhouse_migrate/main.go --last-migration $(BACKUP_LAST_MIGRATION) --user CLICKHOUSE_USER=$(CLICKHOUSE_USER) --password CLICKHOUSE_PASSWORD=$(CLICKHOUSE_PASSWORD) -# -# Default time periods for benchmarking. -# NOTE: These defaults are aligned with the default demo backup (BACKUP_NAME=20260120). -# PMM_DEMO_BENCH_PERIOD_TO may be in the future relative to the current date and is -# intended for benchmarking with demo data that includes timestamps up to 2026-01-21. -# When using a different backup or real data, override these values so they match -# the actual time range of your data to avoid empty benchmark results. -PMM_DEMO_BENCH_PERIOD_FROM ?= 2025-12-27T00:00:00+01:00 -PMM_DEMO_BENCH_PERIOD_TO ?= 2026-01-21T23:59:59+01:00 -# -# Run bench for all main endpoints. -bench: bench-filters bench-report bench-metrics bench-example -# Run bench for getFilters. -bench-filters: - @echo "Running Filters benchmark" - PMM_DEMO_BENCH_PERIOD_FROM="$(PMM_DEMO_BENCH_PERIOD_FROM)" PMM_DEMO_BENCH_PERIOD_TO="$(PMM_DEMO_BENCH_PERIOD_TO)" \ - go test -bench ^BenchmarkGetFilters$$ -run=^$$ -v -benchtime=10x pmm_demo_benchmark_test.go -# Run bench for getReport. -bench-report: - @echo "Running Report benchmark" - PMM_DEMO_BENCH_PERIOD_FROM="$(PMM_DEMO_BENCH_PERIOD_FROM)" PMM_DEMO_BENCH_PERIOD_TO="$(PMM_DEMO_BENCH_PERIOD_TO)" \ - go test -bench ^BenchmarkGetReport$$ -run=^$$ -v -benchtime=10x pmm_demo_benchmark_test.go -# Run bench for getMetrics. -bench-metrics: - @echo "Running Metrics benchmark" - PMM_DEMO_BENCH_PERIOD_FROM="$(PMM_DEMO_BENCH_PERIOD_FROM)" PMM_DEMO_BENCH_PERIOD_TO="$(PMM_DEMO_BENCH_PERIOD_TO)" \ - go test -bench ^BenchmarkGetMetrics$$ -run=^$$ -v -benchtime=10x pmm_demo_benchmark_test.go -# Run bench for getExample. -bench-example: - @echo "Running Example benchmark" - PMM_DEMO_BENCH_PERIOD_FROM="$(PMM_DEMO_BENCH_PERIOD_FROM)" PMM_DEMO_BENCH_PERIOD_TO="$(PMM_DEMO_BENCH_PERIOD_TO)" \ - go test -bench ^BenchmarkGetExample$$ -run=^$$ -v -benchtime=10x pmm_demo_benchmark_test.go diff --git a/qan-api2/docker-compose.yaml b/qan-api2/docker-compose.yaml deleted file mode 100644 index 1a45a46649b..00000000000 --- a/qan-api2/docker-compose.yaml +++ /dev/null @@ -1,148 +0,0 @@ ---- -services: - - pmm-server: - container_name: pmm-server - image: perconalab/pmm-server:3-dev-latest - ports: - - 443:8443 - - 19000:9000 - - 9933:9933 - restart: always - - ch: - image: clickhouse/clickhouse-server:25.3.6.56 - platform: linux/amd64 - container_name: ch-server - environment: - - CLICKHOUSE_USER=default - - CLICKHOUSE_PASSWORD=clickhouse - ulimits: - nofile: - soft: 262144 - hard: 262144 - ports: - - 9000:9000 - - pmm-client: - container_name: pmm-client - image: centos - depends_on: - - pmm-server - - sysbench-ps - command: > - bash -c " - yum install https://repo.percona.com/yum/percona-release-latest.noarch.rpm -y - percona-release disable all - percona-release enable pmm3-client testing - yum update -y - yum install pmm-client -y - pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address=pmm-server:443 --server-insecure-tls --server-username=admin --server-password=admin - nohup pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml & - sleep 60 - pmm-admin add mysql --cluster=cl1 --environment=dev --custom-labels='app=wp1' --username=root --password=secret --query-source=slowlog MySQLSlowLog ps:3306 - pmm-admin add mysql --cluster=cl2 --environment=prod --custom-labels='app=wp2' --username=root --password=secret --query-source=perfschema MySQLPerfSchema ps:3306 - tail -f /dev/null - " - volumes: - - logs-directory:/var/log/mysql - - ps: - image: percona/percona-server:latest - container_name: ps - environment: - - MYSQL_ROOT_PASSWORD=secret - ports: - - 3306:3306 - volumes: - - logs-directory:/var/log/mysql - - ${PWD}/my.cnf:/etc/my.cnf - - ${PWD}/mysql.logrotate:/etc/logrotate.d/mysql - - postgres: - image: postgres - environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=secret - ports: - - "5432:5432" - - mongo: - image: mongo - environment: - - MONGO_INITDB_ROOT_USERNAME=mongo - - MONGO_INITDB_ROOT_PASSWORD=secret - ports: - - "27017:27017" - - sysbench-ps: - container_name: sysbench-ps - image: perconalab/sysbench - depends_on: - - ps - command: > - bash -c " - set -o xtrace - sleep 120 - mysql \ - --host=ps \ - --port=3306 \ - --user=root \ - --password=secret \ - -e 'CREATE DATABASE IF NOT EXISTS sbtest; SET GLOBAL slow_query_log_file = \"/var/log/mysql/mysql-slow.log\"; SET GLOBAL slow_query_log = \"ON\"; SET GLOBAL slow_query_log_always_write_time = 0;' - sysbench \ - --db-driver=mysql \ - --mysql-host=ps \ - --mysql-port=3306 \ - --mysql-user=root \ - --mysql-password=secret \ - --mysql-db=sbtest \ - --mysql-table-engine=innodb \ - --oltp-table-size=1000000 \ - /usr/share/sysbench/tests/include/oltp_legacy/oltp.lua \ - prepare - sysbench \ - --rate=200 \ - --threads=64 \ - --report-interval=10 \ - --time=0 \ - --events=0 \ - --rand-type=pareto \ - --db-driver=mysql \ - --mysql-host=ps \ - --mysql-port=3306 \ - --mysql-user=root \ - --mysql-password=secret \ - --mysql-db=sbtest \ - --mysql-table-engine=innodb \ - --oltp-reconnect \ - --oltp-table-size=1000000 \ - /usr/share/sysbench/tests/include/oltp_legacy/select.lua \ - run - " - - sysbench-pstpcc: - image: perconalab/sysbench - depends_on: - - ps - command: > - bash -c " - set -o xtrace - sleep 200 - mysql \ - --host=ps \ - --port=3306 \ - --user=root \ - --password=secret \ - -e 'CREATE DATABASE IF NOT EXISTS sbtest; SET GLOBAL slow_query_log_file = \"/var/log/mysql/mysql-slow.log\"; SET GLOBAL slow_query_log = \"ON\"; SET GLOBAL slow_query_log_always_write_time = 0;' - cd /sysbench/sysbench-tpcc - ./tpcc.lua --mysql-host=ps --mysql-port=3306 --mysql-user=root --mysql-password=secret --mysql-db=sbtest --time=300 --threads=64 --report-interval=1 --tables=10 --scale=100 --db-driver=mysql prepare - ./tpcc.lua --mysql-host=ps --mysql-port=3306 --mysql-user=root --mysql-password=secret --mysql-db=sbtest --time=300 --threads=64 --report-interval=1 --tables=10 --scale=100 --db-driver=mysql run - " - -volumes: - logs-directory: - driver_opts: - type: volume - device: ${PWD}/logs - o: bind diff --git a/qan-api2/docker-compose.yml b/qan-api2/docker-compose.yml new file mode 100644 index 00000000000..80f98d772b4 --- /dev/null +++ b/qan-api2/docker-compose.yml @@ -0,0 +1,40 @@ +--- +# See usage instructions in the root docker-compose.yml. +services: + + pmm-server: + profiles: + - pmm + container_name: pmm-server + image: perconalab/pmm-server:3-dev-latest + restart: always + depends_on: + ch: + condition: service_healthy + environment: + - PMM_CLICKHOUSE_ADDR=ch:9000 + - PMM_CLICKHOUSE_DATABASE=pmm + - PMM_CLICKHOUSE_USER=default + - PMM_CLICKHOUSE_PASSWORD=clickhouse + - PMM_DISABLE_BUILTIN_CLICKHOUSE=1 + + ch: + profiles: + - pmm + image: clickhouse/clickhouse-server:25.3.6.56 + container_name: ch-server + environment: + - CLICKHOUSE_USER=default + - CLICKHOUSE_PASSWORD=clickhouse + ulimits: + nofile: + soft: 262144 + hard: 262144 + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 10s + ports: + - 9000:9000 diff --git a/ui/docker-compose.yml b/ui/docker-compose.yml index 19b1e57b568..88cb26d1205 100644 --- a/ui/docker-compose.yml +++ b/ui/docker-compose.yml @@ -25,16 +25,14 @@ services: # - ../../grafana-dashboards/pmm-app/dist:/srv/grafana/plugins/pmm-app # Mount repo so you can run the backend built from this codebase # - ..:/root/go/src/github.com/percona/pmm - # - ../Makefile.devcontainer:/root/go/src/github.com/percona/pmm/Makefile:ro + # (no Makefile override needed — root Makefile already includes .devcontainer/Makefile) # should be followed by running: # docker exec -u root -it pmm-server bash -c 'cd /root/go/src/github.com/percona/pmm && make run-agent' # docker exec -u root -it pmm-server bash -c 'cd /root/go/src/github.com/percona/pmm && make run-managed' environment: - PMM_DEBUG=1 - GF_DEFAULT_APP_MODE=development - - PMM_DEV_PORTAL_URL=https://portal-dev.percona.com - PMM_DEV_PERCONA_PLATFORM_ADDRESS=https://check-dev.percona.com:443 - - PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY=RWTkF7Snv08FCboTne4djQfN5qbrLfAjb8SY3/wwEP+X5nUrkxCEvUDJ - PMM_WATCHTOWER_HOST=http://watchtower:8080 - PMM_WATCHTOWER_TOKEN=123 # Uncomment to specify to which docker image to update