PMM-13487 A pipeline to build PMM anywhere

.devcontainer/setup.py

@@ -77,7 +77,10 @@ def setup():
         # Turns fsync off. Create database operations with fsync on are very slow on Ubuntu.
         # Having fsync off in dev environment is fine.
         "sed -i -e \"s/#fsync = on/fsync = off/\" /srv/postgres14/postgresql.conf",
+        # Configure pg_hba.conf for password authentication from all hosts (dev environment only)
+        # Note: In dev, we allow both trust and scram-sha-256 for convenience
         "echo 'host    all         all     0.0.0.0/0     trust' >> /srv/postgres14/pg_hba.conf",
+        "echo 'host    all         all     0.0.0.0/0     scram-sha-256' >> /srv/postgres14/pg_hba.conf",
         # "supervisorctl restart postgresql",
     ])
 

.github/workflows/build-pipeline.yml

@@ -0,0 +1,34 @@
+name: Build Pipeline
+
+on:
+  workflow_dispatch:
+    inputs:
+      target:
+        description: 'Build target: all | client | server'
+        required: true
+        default: 'all'
+        type: choice
+        options:
+          - all
+          - client
+          - server
+      pr_number:
+        description: 'PR number (informational — used for logging)'
+        required: false
+        default: ''
+
+jobs:
+  build:
+    name: Build PMM (${{ github.event.inputs.target }})
+    # Requires a self-hosted runner with Docker + Minio configured.
+    # Adjust the runner label once the build infrastructure is in place.
+    runs-on: [self-hosted, pmm-builder]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build
+        run: |
+          cd build/pipeline
+          make build-${{ github.event.inputs.target }}

.github/workflows/pmm-bot.yml

@@ -0,0 +1,111 @@
+name: PMM Bot
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  dispatch:
+    # Only handle PR comments that start with @pmm-bot
+    if: >
+      github.event.issue.pull_request != null &&
+      startsWith(github.event.comment.body, '@pmm-bot')
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write        # post comments and reactions
+      pull-requests: write
+      actions: write       # trigger workflow_dispatch
+
+    steps:
+      - uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const body        = context.payload.comment.body.trim();
+            const sender      = context.payload.comment.user.login;
+            const owner       = context.repo.owner;
+            const repo        = context.repo.repo;
+            const issueNumber = context.payload.issue.number;
+            const commentId   = context.payload.comment.id;
+
+            // ── 1. Permission check ──────────────────────────────────────────
+            let permission;
+            try {
+              const { data } = await github.rest.repos.getCollaboratorPermissionLevel({
+                owner, repo, username: sender,
+              });
+              permission = data.permission;
+            } catch (e) {
+              // Non-collaborators get a 404 from this endpoint.
+              permission = 'none';
+            }
+
+            if (!['write', 'admin'].includes(permission)) {
+              await github.rest.issues.createComment({
+                owner, repo, issue_number: issueNumber,
+                body: `@${sender} You need write access to trigger bot commands.`,
+              });
+              return;
+            }
+
+            // ── 2. Acknowledge ───────────────────────────────────────────────
+            await github.rest.reactions.createForIssueComment({
+              owner, repo, comment_id: commentId, content: 'eyes',
+            });
+
+            // ── 3. Resolve the PR branch ─────────────────────────────────────
+            const { data: pr } = await github.rest.pulls.get({
+              owner, repo, pull_number: issueNumber,
+            });
+            const ref = pr.head.ref;
+
+            // ── 4. Parse command ─────────────────────────────────────────────
+            // Strip the "@pmm-bot " prefix and normalise whitespace.
+            const cmd = body.replace(/^@pmm-bot\s+/i, '').trim().toLowerCase();
+
+            const COMMANDS = {
+              'rebuild all':    { workflow: 'build-pipeline.yml', inputs: { target: 'all'    } },
+              'rebuild client': { workflow: 'build-pipeline.yml', inputs: { target: 'client' } },
+              'rebuild server': { workflow: 'build-pipeline.yml', inputs: { target: 'server' } },
+              'run api tests':  { workflow: 'api-tests.yml',      inputs: {}                   },
+            };
+
+            const HELP_TEXT =
+              '**Available commands:**\n' +
+              '- `@pmm-bot rebuild all` — build client + server\n' +
+              '- `@pmm-bot rebuild client` — build client only\n' +
+              '- `@pmm-bot rebuild server` — build server only\n' +
+              '- `@pmm-bot run API tests` — run API test suite\n' +
+              '- `@pmm-bot help` — show this message\n';
+
+            if (cmd === 'help') {
+              await github.rest.issues.createComment({
+                owner, repo, issue_number: issueNumber, body: HELP_TEXT,
+              });
+              return;
+            }
+
+            if (!(cmd in COMMANDS)) {
+              await github.rest.issues.createComment({
+                owner, repo, issue_number: issueNumber,
+                body: `Unknown command: \`${cmd}\`\n\n${HELP_TEXT}`,
+              });
+              return;
+            }
+
+            // ── 5. Dispatch ──────────────────────────────────────────────────
+            const { workflow, inputs } = COMMANDS[cmd];
+            await github.rest.actions.createWorkflowDispatch({
+              owner, repo,
+              workflow_id: workflow,
+              ref,
+              inputs: { ...inputs, pr_number: String(issueNumber) },
+            });
+
+            // ── 6. Confirm ───────────────────────────────────────────────────
+            const runsUrl =
+              `https://github.com/${owner}/${repo}/actions/workflows/${workflow}`;
+            await github.rest.issues.createComment({
+              owner, repo, issue_number: issueNumber,
+              body: `@${sender} Dispatched \`${cmd}\` on branch \`${ref}\`. [View runs](${runsUrl})`,
+            });

.gitignore

@@ -2,6 +2,7 @@
 /.vscode/
 *.iml
 
+# Temprorary build files
 /api/nginx/*.pem
 bin/
 !/documentation/resources/bin/
@@ -13,7 +14,6 @@ cover.out
 crosscover.out
 
 agent/testdata/
-
 agent/agents/mysql/slowlog/parser/corpus/
 agent/agents/mysql/slowlog/parser/crashers/
 agent/agents/mysql/slowlog/parser/suppressions/
@@ -32,22 +32,22 @@ compose.yml
 # ViM temporary files
 *.sw[o,p]
 
+# build ops
 .env
 .netrc
 .modules
 build.log
+packer.log
 ci.yml
 
-api-tests/pmm-api-tests-output.txt
-api-tests/pmm-api-tests-junit-report.xml
+pmm-api-tests-output.txt
+pmm-api-tests-junit-report.xml
 
-packer.log
 encryption.key
 
 /tmp/
 settings.json
 GEMINI.md
-RTA-Jira-Tasks-Summary.md
 
 # PMM Demo backups (it would increase repository size significantly)
 dev/clickhouse-backups/

build/ansible/roles/pmm-images/tasks/main.yml

@@ -10,15 +10,6 @@
 - name: List installed gpg keys
   command: ls -la /etc/pki/rpm-gpg
 
-# Local yum repo for building pmm server docker image in autobuild jobs
-- name: Add a local YUM repository
-  yum_repository:
-    name: local
-    description: Local YUM repository - x86_64
-    baseurl: file:///tmp/RPMS
-    gpgcheck: no
-    enabled: no
-
 - name: Update OS packages
   dnf:
     name: "*"
@@ -96,18 +87,110 @@
     - /var/lib/cloud/scripts/per-once
     - /var/lib/cloud/scripts/per-boot
 
-- name: Install PMM Server components
-  dnf:
-    name:
-      - percona-grafana
-      - percona-victoriametrics
-      - percona-qan-api2
-      - percona-dashboards
-      - pmm-managed
-      - pmm-dump
-      - vmproxy
-    state: installed
-    enablerepo: local
+- name: Create server component directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: pmm
+    group: root
+    mode: 0775
+  loop:
+    - /usr/share/grafana
+    - /usr/share/pmm-managed
+    - /usr/share/pmm-ui
+    - /usr/share/percona-dashboards
+    - /usr/share/percona-dashboards/panels
+    - /usr/local/percona
+    - /usr/local/percona/advisors
+    - /usr/local/percona/checks
+    - /usr/local/percona/alerting-templates
+    - /var/lib/grafana
+
+- name: Install Go binaries
+  copy:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: root
+    group: root
+    mode: "0755"
+    remote_src: yes
+  loop:
+    - { src: /opt/pmm-staging/pmm-managed/bin/pmm-managed, dest: /usr/sbin/pmm-managed }
+    - { src: /opt/pmm-staging/pmm-managed/bin/pmm-encryption-rotation, dest: /usr/sbin/pmm-encryption-rotation }
+    - { src: /opt/pmm-staging/pmm-managed/bin/pmm-managed-init, dest: /usr/sbin/pmm-managed-init }
+    - { src: /opt/pmm-staging/pmm-managed/bin/pmm-managed-starlark, dest: /usr/sbin/pmm-managed-starlark }
+    - { src: /opt/pmm-staging/pmm-managed/bin/qan-api2, dest: /usr/sbin/percona-qan-api2 }
+    - { src: /opt/pmm-staging/pmm-managed/bin/vmproxy, dest: /usr/sbin/vmproxy }
+    - { src: /opt/pmm-staging/pmm-dump/pmm-dump, dest: /usr/sbin/pmm-dump }
+    - { src: /opt/pmm-staging/grafana-go/grafana-server, dest: /usr/sbin/grafana-server }
+    - { src: /opt/pmm-staging/grafana-go/grafana, dest: /usr/sbin/grafana }
+    - { src: /opt/pmm-staging/grafana-go/grafana-cli, dest: /usr/bin/grafana-cli }
+    - { src: /opt/pmm-staging/victoriametrics/victoria-metrics-pure, dest: /usr/sbin/victoriametrics }
+    - { src: /opt/pmm-staging/victoriametrics/vmalert-pure, dest: /usr/sbin/vmalert }
+
+- name: Install Grafana UI assets
+  command: cp -rT /opt/pmm-staging/grafana-ui/{{ item.src }} /usr/share/grafana/{{ item.dest }}
+  loop:
+    - { src: public, dest: public }
+    - { src: conf, dest: conf }
+    - { src: tools, dest: tools }
+
+- name: Install Grafana configuration files
+  copy:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: pmm
+    group: root
+    mode: "0644"
+    remote_src: yes
+  loop:
+    - { src: /opt/pmm-staging/grafana-ui/conf/sample.ini, dest: /etc/grafana/grafana.ini }
+    - { src: /opt/pmm-staging/grafana-ui/conf/ldap.toml, dest: /etc/grafana/ldap.toml }
+
+- name: Install pmm-managed data assets
+  command: cp -rT /opt/pmm-staging/pmm-managed/data/{{ item.src }} {{ item.dest }}
+  loop:
+    - { src: swagger, dest: /usr/share/pmm-managed/swagger }
+    - { src: advisors, dest: /usr/local/percona/advisors }
+    - { src: checks, dest: /usr/local/percona/checks }
+    - { src: alerting-templates, dest: /usr/local/percona/alerting-templates }
+
+- name: Install PMM UI assets
+  command: cp -rT /opt/pmm-staging/pmm-ui/{{ item.src }} {{ item.dest }}
+  loop:
+    - { src: pmm-dist, dest: /usr/share/pmm-ui }
+    - { src: pmm-compat-dist, dest: /usr/share/percona-dashboards/panels/pmm-compat-app }
+
+- name: Install percona-dashboards assets
+  command: cp -rT /opt/pmm-staging/pmm-dashboards/{{ item.src }} {{ item.dest }}
+  loop:
+    - { src: panels, dest: /usr/share/percona-dashboards/panels }
+    - { src: pmm-app-dist, dest: /usr/share/percona-dashboards/panels/pmm-app }
+
+- name: Install VERSION.json
+  copy:
+    src: /opt/pmm-staging/version.json
+    dest: /usr/share/pmm-server/VERSION.json
+    owner: root
+    group: root
+    mode: "0644"
+    remote_src: yes
+
+- name: Set ownership on pmm-owned assets
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: pmm
+    group: root
+    recurse: yes
+  loop:
+    - /usr/share/grafana
+    - /usr/share/pmm-managed
+    - /usr/share/pmm-ui
+    - /usr/share/percona-dashboards
+    - /usr/local/percona
+
+
 
 - name: Configure grafana
   include_role:

build/docker/server/Dockerfile.el9

@@ -19,7 +19,8 @@ RUN --mount=type=cache,target=/var/cache/dnf \
                         ansible-collection-ansible-posix \
                         glibc-langpack-en \
                         dnf \
-                        vi
+                        vi && \
+                        dnf -y remove microdnf
 
 COPY entrypoint.sh /opt/entrypoint.sh
 COPY ansible /opt/ansible