From 0390c88417801b9355725a1f83d5e06fae66af65 Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Sun, 15 Mar 2026 02:06:36 +0100 Subject: [PATCH 1/9] PG-2029 Introduce Meson build system To give a more modern development expereince we plan to replace our current make and PGXS based build system with one which uses Meson and pg_config to achieve the same thing. Potential advantages: - More modern and pleasant expereince - Parallel tests - Allows for Windows support Potential disadvantages: - We may have to manually implement support for odd platforms --- meson.build | 160 ++++++++++++++++++++++++++++++++++++++++++++++ meson_options.txt | 2 + 2 files changed, 162 insertions(+) create mode 100644 meson.build create mode 100644 meson_options.txt diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..095b5618b --- /dev/null +++ b/meson.build @@ -0,0 +1,160 @@ +project('pg_tde', + ['c'], + version: '2.1.2', + meson_version: '>=0.53', +) + +pg_config = find_program(get_option('pg_config')) + +bindir = run_command(pg_config, '--bindir', check: true).stdout().strip() +includedir_server = run_command(pg_config, '--includedir-server', check: true).stdout().strip() +libdir = run_command(pg_config, '--libdir', check: true).stdout().strip() +pkglibdir = run_command(pg_config, '--pkglibdir', check: true).stdout().strip() +sharedir = run_command(pg_config, '--sharedir', check: true).stdout().strip() +version = run_command(pg_config, '--version', check: true).stdout().strip() + +cc = meson.get_compiler('c') + +# To make sure we compile code taken from PostgreSQL the safely we need to use +# the same fucntional flags. +# +# These are copied from common_functional_flags in meson.build in PostgreSQL 18. +add_project_arguments(cc.get_supported_arguments([ + '-fno-strict-aliasing', + '-fwrapv', + '-fexcess-precision=standard', +]), language: ['c']) + +# TODO: Why does it seems like they sometimes are in pkglibdir and other times in libdir? +pgport = cc.find_library('pgport', dirs: [pkglibdir, libdir], static: true) +pgcommon = cc.find_library('pgcommon', dirs: [pkglibdir, libdir], static: true) +pgfeutils = cc.find_library('pgfeutils', dirs: [pkglibdir, libdir], static: true) + +crypto = dependency('libcrypto') +ssl = dependency('libssl') +curl = dependency('libcurl') +lz4 = dependency('liblz4', required: false) +z = dependency('zlib') +zstd = dependency('libzstd', required: false) + +incdirs = include_directories( + 'src/include', + 'src/libkmip/libkmip/include', + includedir_server, +) + +install_data( + 'pg_tde.control', + 'pg_tde--1.0.sql', + 'pg_tde--1.0--2.0.sql', + 'pg_tde--2.0--2.1.sql', + install_dir: sharedir / 'extension', +) + +kmip = static_library('kmip', + files( + 'src/libkmip/libkmip/src/kmip.c', + 'src/libkmip/libkmip/src/kmip_bio.c', + 'src/libkmip/libkmip/src/kmip_locate.c', + 'src/libkmip/libkmip/src/kmip_memset.c', + ), + c_args: ['-w'], # This is a 3rd party, disable warnings completely + include_directories: incdirs, +) + +shared_module('pg_tde', + files( + 'src/access/pg_tde_tdemap.c', + 'src/access/pg_tde_xlog.c', + 'src/access/pg_tde_xlog_keys.c', + 'src/access/pg_tde_xlog_smgr.c', + 'src/catalog/tde_keyring.c', + 'src/catalog/tde_keyring_parse_opts.c', + 'src/catalog/tde_principal_key.c', + 'src/common/pg_tde_utils.c', + 'src/encryption/enc_aes.c', + 'src/encryption/enc_tde.c', + 'src/keyring/keyring_api.c', + 'src/keyring/keyring_curl.c', + 'src/keyring/keyring_file.c', + 'src/keyring/keyring_kmip.c', + 'src/keyring/keyring_kmip_impl.c', + 'src/keyring/keyring_vault.c', + 'src/pg_tde.c', + 'src/pg_tde_event_capture.c', + 'src/pg_tde_guc.c', + 'src/smgr/pg_tde_smgr.c', + ), + include_directories: incdirs, + install: true, + install_dir: pkglibdir, + dependencies: [crypto, ssl, curl], + name_prefix: '', + link_with: [kmip], +) + +pg_tde_frontend = static_library('pg_tde_frontend', + files( + 'src/access/pg_tde_tdemap.c', + 'src/access/pg_tde_xlog_keys.c', + 'src/catalog/tde_keyring.c', + 'src/catalog/tde_keyring_parse_opts.c', + 'src/catalog/tde_principal_key.c', + 'src/common/pg_tde_utils.c', + 'src/encryption/enc_aes.c', + 'src/encryption/enc_tde.c', + 'src/keyring/keyring_api.c', + 'src/keyring/keyring_curl.c', + 'src/keyring/keyring_file.c', + 'src/keyring/keyring_kmip.c', + 'src/keyring/keyring_kmip_impl.c', + 'src/keyring/keyring_vault.c', + ), + c_args: ['-DFRONTEND'], + dependencies: [pgfeutils, pgcommon, pgport, curl, crypto, ssl], + include_directories: incdirs, + link_with: [kmip], +) + +pg_tde_frontend_xlog = static_library('pg_tde_frontend_xlog', + files( + 'src/access/pg_tde_xlog_smgr.c', + fetools / 'xlogreader.c', + ), + dependencies: [lz4, zstd], + c_args: ['-DFRONTEND'], + include_directories: incdirs, + link_with: [kmip], +) + +executable('pg_tde_change_key_provider', + files('src/bin/pg_tde_change_key_provider.c'), + include_directories: incdirs, + install: true, + install_dir: bindir, + link_with: [pg_tde_frontend], +) + +executable('pg_tde_archive_decrypt', + files('src/bin/pg_tde_archive_decrypt.c'), + include_directories: incdirs, + install: true, + install_dir: bindir, + link_with: [pg_tde_frontend, pg_tde_frontend_xlog], +) + +executable('pg_tde_restore_encrypt', + files('src/bin/pg_tde_restore_encrypt.c'), + include_directories: incdirs, + install: true, + install_dir: bindir, + link_with: [pg_tde_frontend, pg_tde_frontend_xlog], +) + +executable('pg_tde_upgrade', + files('src/bin/pg_tde_upgrade.c'), + include_directories: incdirs, + install: true, + install_dir: bindir, + dependencies: [pgcommon, pgport], +) diff --git a/meson_options.txt b/meson_options.txt new file mode 100644 index 000000000..6987c0bd3 --- /dev/null +++ b/meson_options.txt @@ -0,0 +1,2 @@ +option('pg_config', type: 'string', value: 'pg_config', + description: 'Path to pg_config executable') From 0555710fdb0e04980e5060925e22867ceda60d39 Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Sun, 15 Mar 2026 05:04:40 +0100 Subject: [PATCH 2/9] PG-2029 Add support for building frontend tools with Meson This is split out from the previous commit to amke the history easier to read and the commits easier to review. --- meson.build | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/meson.build b/meson.build index 095b5618b..f8d46a75e 100644 --- a/meson.build +++ b/meson.build @@ -7,8 +7,10 @@ project('pg_tde', pg_config = find_program(get_option('pg_config')) bindir = run_command(pg_config, '--bindir', check: true).stdout().strip() +includedir = run_command(pg_config, '--includedir', check: true).stdout().strip() includedir_server = run_command(pg_config, '--includedir-server', check: true).stdout().strip() libdir = run_command(pg_config, '--libdir', check: true).stdout().strip() +pkgincludedir = run_command(pg_config, '--pkgincludedir', check: true).stdout().strip() pkglibdir = run_command(pg_config, '--pkglibdir', check: true).stdout().strip() sharedir = run_command(pg_config, '--sharedir', check: true).stdout().strip() version = run_command(pg_config, '--version', check: true).stdout().strip() @@ -30,6 +32,7 @@ pgport = cc.find_library('pgport', dirs: [pkglibdir, libdir], static: true) pgcommon = cc.find_library('pgcommon', dirs: [pkglibdir, libdir], static: true) pgfeutils = cc.find_library('pgfeutils', dirs: [pkglibdir, libdir], static: true) +pq = cc.find_library('pq', dirs: [libdir]) crypto = dependency('libcrypto') ssl = dependency('libssl') curl = dependency('libcurl') @@ -37,10 +40,17 @@ lz4 = dependency('liblz4', required: false) z = dependency('zlib') zstd = dependency('libzstd', required: false) +major_version = version.split(' ')[1].split('.')[0].to_int() + +fetools = 'fetools/pg@0@'.format(major_version) + incdirs = include_directories( 'src/include', 'src/libkmip/libkmip/include', + fetools / 'include', includedir_server, + includedir, + pkgincludedir / 'internal', ) install_data( @@ -158,3 +168,109 @@ executable('pg_tde_upgrade', install_dir: bindir, dependencies: [pgcommon, pgport], ) + +if major_version == 18 + bb_sources = files( + fetools / 'pg_basebackup/receivelog.c', + fetools / 'pg_basebackup/streamutil.c', + fetools / 'pg_basebackup/walmethods.c', + fetools / 'pg_basebackup/pg_basebackup.c', + fetools / 'pg_basebackup/astreamer_inject.c', + ) +else + bb_sources = files( + fetools / 'pg_basebackup/receivelog.c', + fetools / 'pg_basebackup/streamutil.c', + fetools / 'pg_basebackup/walmethods.c', + fetools / 'pg_basebackup/pg_basebackup.c', + fetools / 'pg_basebackup/bbstreamer_file.c', + fetools / 'pg_basebackup/bbstreamer_gzip.c', + fetools / 'pg_basebackup/bbstreamer_inject.c', + fetools / 'pg_basebackup/bbstreamer_lz4.c', + fetools / 'pg_basebackup/bbstreamer_tar.c', + fetools / 'pg_basebackup/bbstreamer_zstd.c', + ) +endif + +executable('pg_tde_basebackup', + bb_sources, + include_directories: incdirs, + install: true, + install_dir: bindir, + dependencies: [pq, z], + link_with: [pg_tde_frontend, pg_tde_frontend_xlog], +) + +executable('pg_tde_checksums', + files( + fetools / 'pg_checksums/pg_checksums.c', + ), + include_directories: incdirs, + install: true, + install_dir: bindir, + link_with: [pg_tde_frontend], +) + +executable('pg_tde_resetwal', + files( + fetools / 'pg_resetwal/pg_resetwal.c', + ), + include_directories: incdirs, + install: true, + install_dir: bindir, + link_with: [pg_tde_frontend, pg_tde_frontend_xlog], +) + +executable('pg_tde_rewind', + files( + fetools / 'pg_rewind/datapagemap.c', + fetools / 'pg_rewind/file_ops.c', + fetools / 'pg_rewind/filemap.c', + fetools / 'pg_rewind/libpq_source.c', + fetools / 'pg_rewind/local_source.c', + fetools / 'pg_rewind/parsexlog.c', + fetools / 'pg_rewind/pg_rewind.c', + fetools / 'pg_rewind/timeline.c', + ), + include_directories: incdirs, + install: true, + install_dir: bindir, + dependencies: [pq], + link_with: [pg_tde_frontend, pg_tde_frontend_xlog], +) + +executable('pg_tde_waldump', + files( + fetools / 'pg_waldump/compat.c', + fetools / 'pg_waldump/pg_waldump.c', + fetools / 'pg_waldump/rmgrdesc.c', + fetools / 'rmgrdesc/brindesc.c', + fetools / 'rmgrdesc/clogdesc.c', + fetools / 'rmgrdesc/committsdesc.c', + fetools / 'rmgrdesc/dbasedesc.c', + fetools / 'rmgrdesc/genericdesc.c', + fetools / 'rmgrdesc/gindesc.c', + fetools / 'rmgrdesc/gistdesc.c', + fetools / 'rmgrdesc/hashdesc.c', + fetools / 'rmgrdesc/heapdesc.c', + fetools / 'rmgrdesc/logicalmsgdesc.c', + fetools / 'rmgrdesc/mxactdesc.c', + fetools / 'rmgrdesc/nbtdesc.c', + fetools / 'rmgrdesc/relmapdesc.c', + fetools / 'rmgrdesc/replorigindesc.c', + fetools / 'rmgrdesc/rmgrdesc_utils.c', + fetools / 'rmgrdesc/seqdesc.c', + fetools / 'rmgrdesc/smgrdesc.c', + fetools / 'rmgrdesc/spgdesc.c', + fetools / 'rmgrdesc/standbydesc.c', + fetools / 'rmgrdesc/tblspcdesc.c', + fetools / 'rmgrdesc/xactdesc.c', + fetools / 'rmgrdesc/xlogdesc.c', + fetools / 'xlogstats.c', + ), + include_directories: incdirs, + install: true, + install_dir: bindir, + c_args: ['-DFRONTEND'], + link_with: [pg_tde_frontend, pg_tde_frontend_xlog], +) From b6e95d8ebde13b57a4b16ffaf40015f3f82c5d03 Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Sun, 15 Mar 2026 05:05:29 +0100 Subject: [PATCH 3/9] PG-2029 Add support for pg_regress tests in Meson One of the weird things with the Meson tests for pg_tde is that they are actually install tests so we need to manually run meson install before running meson test. --- meson.build | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/meson.build b/meson.build index f8d46a75e..54b281990 100644 --- a/meson.build +++ b/meson.build @@ -274,3 +274,34 @@ executable('pg_tde_waldump', c_args: ['-DFRONTEND'], link_with: [pg_tde_frontend, pg_tde_frontend_xlog], ) + +regress_tests = [ + 'access_control', + 'alter_index', + 'change_access_method', + 'create_database', + 'default_principal_key', + 'delete_principal_key', + 'insert_update_delete', + 'key_provider', + 'kmip_test', + 'partition_table', + 'pg_tde_is_encrypted', + 'recreate_storage', + 'relocate', + 'tablespace', + 'toast_decrypt', + 'vault_v2_test', + 'version', +] + +pg_regress = find_program('pg_regress', dirs: [pkglibdir / 'pgxs/src/test/regress']) + +test('regress', + pg_regress, + protocol: 'tap', + args: [ + '--bindir', bindir, + '--inputdir', meson.current_source_dir(), + ] + regress_tests, +) From e3d3bd3a8cfc1d8115c6f317cd152e5ff6ab6101 Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Sun, 15 Mar 2026 05:05:07 +0100 Subject: [PATCH 4/9] PG-2029 Add support for TAP tests in Meson One of the nice advantages of meson is that test can be run in parallel which we now can take advantage of. --- meson.build | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++- testwrap | 18 +++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100755 testwrap diff --git a/meson.build b/meson.build index 54b281990..65145deab 100644 --- a/meson.build +++ b/meson.build @@ -1,9 +1,11 @@ project('pg_tde', ['c'], version: '2.1.2', - meson_version: '>=0.53', + meson_version: '>=0.56', ) +fs = import('fs') + pg_config = find_program(get_option('pg_config')) bindir = run_command(pg_config, '--bindir', check: true).stdout().strip() @@ -295,6 +297,43 @@ regress_tests = [ 'version', ] +tap_tests = [ + 't/2pc_replication.pl', + 't/basic.pl', + 't/change_key_provider.pl', + 't/crash_recovery.pl', + 't/key_rotate_tablespace.pl', + 't/keys_update.pl', + 't/key_validation.pl', + 't/multiple_extensions.pl', + 't/pg_basebackup.pl', + 't/pg_resetwal_basic.pl', + 't/pg_resetwal_corrupted.pl', + 't/pg_rewind_basic.pl', + 't/pg_rewind_databases.pl', + 't/pg_rewind_extrafiles.pl', + 't/pg_rewind_growing_files.pl', + 't/pg_rewind_keep_recycled_wals.pl', + 't/pg_rewind_min_recovery_point.pl', + 't/pg_rewind_options.pl', + 't/pg_rewind_pg_xlog_symlink.pl', + 't/pg_rewind_same_timeline.pl', + 't/pg_tde_change_key_provider.pl', + 't/pg_tde_upgrade.pl', + 't/pg_waldump_basic.pl', + 't/pg_waldump_fullpage.pl', + 't/replication.pl', + 't/reuse_relfilenode_in_cache.pl', + 't/rotate_key.pl', + 't/stream_rep.pl', + 't/tde_heap_aes_256.pl', + 't/tde_heap.pl', + 't/unlogged_tables.pl', + 't/wal_archiving.pl', + 't/wal_encrypt.pl', + 't/wal_key_tli.pl', +] + pg_regress = find_program('pg_regress', dirs: [pkglibdir / 'pgxs/src/test/regress']) test('regress', @@ -305,3 +344,27 @@ test('regress', '--inputdir', meson.current_source_dir(), ] + regress_tests, ) + +testwrap = files('testwrap') +perl = find_program('perl') +pg_regress = find_program('pg_regress', dirs: [pkglibdir / 'pgxs/src/test/regress']) + +env = environment() +env.prepend('PATH', bindir) +env.set('PG_REGRESS', pg_regress.full_path()) + +foreach test : tap_tests + test(test, + testwrap, + protocol: 'tap', + env: env, + args: [ + fs.stem(fs.name(test)), + meson.project_source_root(), + perl.full_path(), + '-I', meson.project_source_root() / 't', + '-I', pkglibdir / 'pgxs/src/test/perl', + files(test), + ], + ) +endforeach diff --git a/testwrap b/testwrap new file mode 100755 index 000000000..4bda189b0 --- /dev/null +++ b/testwrap @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +TEST=$1 +SRCDIR=$2 + +mkdir -p testrun +rm -rf "testrun/$TEST" +mkdir "testrun/$TEST" + +export TESTDATADIR=$(pwd)/testrun/$TEST/data +export TESTLOGDIR=$(pwd)/testrun/$TEST/log + +cd "$SRCDIR" + +shift 2 +exec $@ From f9cf1c28a1c8f50c9cadc5377b9cdf1e39dada17 Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Fri, 27 Mar 2026 15:22:19 +0100 Subject: [PATCH 5/9] PG-2029 Add MacOS support to Meson builds Based on Makefile.darwin in the PostgreSQL project. --- meson.build | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/meson.build b/meson.build index 65145deab..a35bbd52f 100644 --- a/meson.build +++ b/meson.build @@ -55,6 +55,12 @@ incdirs = include_directories( pkgincludedir / 'internal', ) +backend_link_args = [] + +if host_machine.system() == 'darwin' + backend_link_args += ['-bundle_loader', bindir / 'postgres'] +endif + install_data( 'pg_tde.control', 'pg_tde--1.0.sql', @@ -103,6 +109,7 @@ shared_module('pg_tde', dependencies: [crypto, ssl, curl], name_prefix: '', link_with: [kmip], + link_args: backend_link_args, ) pg_tde_frontend = static_library('pg_tde_frontend', From b382b8175473442d0403f6e7774b9f954469751f Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Sun, 15 Mar 2026 05:12:03 +0100 Subject: [PATCH 6/9] PG-2029 Use Meson instead of make in our CI Since we plan to only support Meson builds in the future let's move all of our CI jobs over to using Meson. For simplicty we put the build directory outside of the git repository. Also this apparently this fixed a bug in our code formatting job for the frotnend tools so let's also commit the changed formatting. Disable the test timeout in meson since the default timeout is an issue when running with sanitizers. --- .github/workflows/build-and-test.yml | 14 +++++------ .github/workflows/coverage.yml | 22 ++++++++-------- .github/workflows/sanitizers.yml | 12 ++++----- ci_scripts/build-and-install-psp.sh | 2 +- ci_scripts/build.sh | 25 +++++++++++++------ ci_scripts/dump-typedefs.sh | 4 +-- ci_scripts/macos-deps.sh | 3 ++- ci_scripts/test.sh | 4 +-- ci_scripts/ubuntu-deps.sh | 2 ++ fetools/pg18/pg_basebackup/astreamer_inject.c | 2 +- 10 files changed, 51 insertions(+), 39 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 9120d6a17..7d4cba977 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -60,7 +60,7 @@ jobs: run: src/ci_scripts/build.sh ${{ inputs.build_type }} - name: Add build and install to artifact tar file - run: tar -czf artifacts.tar src postgres pginst + run: tar -czf artifacts.tar src build postgres pginst - name: Upload build artifacts uses: actions/upload-artifact@v7 @@ -113,13 +113,13 @@ jobs: with: name: log-test-${{ inputs.pg_version }}-${{ inputs.os }}-${{ inputs.compiler }}-${{ inputs.build_type }} path: | - src/regress_install - src/regress_install.log - src/regression.diffs - src/regression.out - src/results + build/meson-logs/testlog.txt + build/regress_install + build/regress_install.log + build/regression.diffs + build/results + build/testrun src/t/results - src/tmp_check retention-days: 3 test-psp-with-tde: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ffb8d45d8..a92406fff 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -47,7 +47,7 @@ jobs: run: src/ci_scripts/build-and-install-psp.sh coverage - name: Build pg_tde - run: src/ci_scripts/build.sh debug + run: src/ci_scripts/build.sh coverage - name: Setup kmip and vault run: src/ci_scripts/setup-keyring-servers.sh @@ -57,9 +57,9 @@ jobs: - name: Process coverage run: | - geninfo -o coverage.info --no-external --rc lcov_branch_coverage=1 -i . - geninfo -o coverage.info --no-external --rc lcov_branch_coverage=1 . - working-directory: src + geninfo -o coverage.info --rc lcov_branch_coverage=1 -i . + geninfo -o coverage.info --rc lcov_branch_coverage=1 . + working-directory: build - name: Upload coverage data to codecov.io uses: codecov/codecov-action@v5 @@ -68,7 +68,7 @@ jobs: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} disable_search: true - files: coverage.info + files: ../build/coverage.info working-directory: src - name: Report on test fail @@ -77,11 +77,11 @@ jobs: with: name: coverage-testlog path: | - src/regress_install - src/regress_install.log - src/regression.diffs - src/regression.out - src/results + build/meson-logs/testlog.txt + build/regress_install + build/regress_install.log + build/regression.diffs + build/results + build/testrun src/t/results - src/tmp_check retention-days: 3 diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 5d6635432..96e402c69 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -66,11 +66,11 @@ jobs: name: sanitizers-testlog path: | sanitize.log.* - src/regress_install - src/regress_install.log - src/regression.diffs - src/regression.out - src/results + build/meson-logs/testlog.txt + build/regress_install + build/regress_install.log + build/regression.diffs + build/results + build/testrun src/t/results - src/tmp_check retention-days: 3 diff --git a/ci_scripts/build-and-install-psp.sh b/ci_scripts/build-and-install-psp.sh index 4883df2ed..6043c662d 100755 --- a/ci_scripts/build-and-install-psp.sh +++ b/ci_scripts/build-and-install-psp.sh @@ -37,7 +37,7 @@ case "$1" in *) echo "Unknown build type: $1" - echo "Please use one of the following: debug, debugoptimized, sanitize" + echo "Please use one of the following: debug, debugoptimized, coverage, sanitize" exit 1 ;; esac diff --git a/ci_scripts/build.sh b/ci_scripts/build.sh index 433f9e76d..836f11543 100755 --- a/ci_scripts/build.sh +++ b/ci_scripts/build.sh @@ -4,31 +4,40 @@ set -e SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" PG_CONFIG="$SCRIPT_DIR/../../pginst/bin/pg_config" -CFLAGS=-Werror - -cd "$SCRIPT_DIR/.." +BUILD_TYPE= +ARGS= case "$1" in debug) echo "Building with debug option" + BUILD_TYPE=$1 ;; debugoptimized) echo "Building with debugoptimized option" - CFLAGS+=" -O2" + BUILD_TYPE=$1 + ;; + + coverage) + echo "Building with coverage option" + BUILD_TYPE=debug + ARGS+=-Db_coverage=true ;; sanitize) echo "Building with sanitize option" - CFLAGS+=" -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer -fno-inline-functions" + BUILD_TYPE=debug + ARGS+=" -Dc_args=['-fsanitize=address','-fsanitize=undefined','-fno-omit-frame-pointer','-fno-inline-functions']" + ARGS+=" -Dc_link_args=['-fsanitize=address','-fsanitize=undefined']" ;; *) echo "Unknown build type: $1" - echo "Please use one of the following: debug, debugoptimized, sanitize" + echo "Please use one of the following: debug, debugoptimized, coverage, sanitize" exit 1 ;; esac -export CFLAGS -make PG_CONFIG="$PG_CONFIG" install -j +cd "$SCRIPT_DIR/.." +meson setup --buildtype="$BUILD_TYPE" -Dpg_config="$PG_CONFIG" -Dwerror=true $ARGS ../build +meson install -C ../build diff --git a/ci_scripts/dump-typedefs.sh b/ci_scripts/dump-typedefs.sh index 3a3d0d7f5..ea650a0ca 100755 --- a/ci_scripts/dump-typedefs.sh +++ b/ci_scripts/dump-typedefs.sh @@ -8,7 +8,7 @@ set -e SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" -cd "$SCRIPT_DIR/.." +cd "$SCRIPT_DIR/../../build" if ! test -f pg_tde.so; then echo "pg_tde.so doesn't exists, run build.sh first in debug mode" @@ -19,4 +19,4 @@ fi ../postgres/src/tools/find_typedef . wget -q -O - "https://buildfarm.postgresql.org/cgi-bin/typedefs.pl?branch=REL_17_STABLE" wget -q -O - "https://buildfarm.postgresql.org/cgi-bin/typedefs.pl?branch=REL_18_STABLE" -) | sort -u > typedefs.list +) | sort -u > ../src/typedefs.list diff --git a/ci_scripts/macos-deps.sh b/ci_scripts/macos-deps.sh index c7510fa72..bed395281 100755 --- a/ci_scripts/macos-deps.sh +++ b/ci_scripts/macos-deps.sh @@ -15,7 +15,8 @@ DEPS=( lz4 openssl zstd - + # Build pg_tde + meson # Run pgperltidy perltidy ) diff --git a/ci_scripts/test.sh b/ci_scripts/test.sh index 0f1dac932..6f01eefca 100755 --- a/ci_scripts/test.sh +++ b/ci_scripts/test.sh @@ -4,7 +4,7 @@ set -e SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" -cd "$SCRIPT_DIR/.." +cd "$SCRIPT_DIR/../../build" OPTS='--set shared_preload_libraries=pg_tde' @@ -16,6 +16,6 @@ fi ../pginst/bin/pg_ctl -D regress_install -l regress_install.log start -make PG_CONFIG=../pginst/bin/pg_config installcheck +meson test --timeout-multiplier=0 --print-errorlogs ../pginst/bin/pg_ctl -D regress_install stop diff --git a/ci_scripts/ubuntu-deps.sh b/ci_scripts/ubuntu-deps.sh index b9b28a8b7..421bc81b6 100755 --- a/ci_scripts/ubuntu-deps.sh +++ b/ci_scripts/ubuntu-deps.sh @@ -40,6 +40,8 @@ DEPS=( xsltproc zlib1g-dev zstd + # pg_tde dependencies + meson # pg_tde test dependencies lcov perltidy diff --git a/fetools/pg18/pg_basebackup/astreamer_inject.c b/fetools/pg18/pg_basebackup/astreamer_inject.c index 7c1e86047..5c713f316 100644 --- a/fetools/pg18/pg_basebackup/astreamer_inject.c +++ b/fetools/pg18/pg_basebackup/astreamer_inject.c @@ -253,7 +253,7 @@ typedef struct astreamer_pg_tde_injector astreamer base; bool skip_file; bool encryped_wal; -} astreamer_pg_tde_injector; +} astreamer_pg_tde_injector; static void astreamer_pg_tde_injector_content(astreamer *streamer, astreamer_member *member, From 500cf902b05eb17db87f81bcbcce54813b7e2c0f Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Tue, 31 Mar 2026 12:39:25 +0200 Subject: [PATCH 7/9] PG-2029 Add simple test to CI for builds using make While we are moving our development builds and CI directly from make to Meson our QA farm and packaging needs more time to migrate to Meson, so to make sure our make file does not break we add a simple CI job which builds pg_tde using make and then runs the test cases. --- .github/workflows/make.yml | 68 ++++++++++++++++++++++++++++++++++++++ ci_scripts/make-build.sh | 11 ++++++ ci_scripts/make-test.sh | 17 ++++++++++ 3 files changed, 96 insertions(+) create mode 100644 .github/workflows/make.yml create mode 100755 ci_scripts/make-build.sh create mode 100755 ci_scripts/make-test.sh diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml new file mode 100644 index 000000000..c483a7277 --- /dev/null +++ b/.github/workflows/make.yml @@ -0,0 +1,68 @@ +name: Build and test with make +on: + pull_request: + paths-ignore: + - documentation/** + push: + branches: + - main + paths-ignore: + - documentation/** + +env: + pg_version: 18 + +jobs: + run: + name: Run + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Clone repository + uses: actions/checkout@v5 + with: + path: src + submodules: recursive + + - name: Clone postgres repository + uses: actions/checkout@v5 + with: + path: postgres + repository: percona/postgres.git + ref: PSP_REL_${{ env.pg_version }}_STABLE + + # KMIP server don't support Python 3.12 for now: https://github.com/OpenKMIP/PyKMIP/pull/707 + - name: Downgrade python to 3.11 + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: Install dependencies + run: src/ci_scripts/ubuntu-deps.sh + + - name: Build postgres + run: src/ci_scripts/build-and-install-psp.sh debugoptimized + + - name: Build pg_tde + run: src/ci_scripts/make-build.sh + + - name: Setup kmip and vault + run: src/ci_scripts/setup-keyring-servers.sh + + - name: Run pg_tde tests + run: src/ci_scripts/make-test.sh + + - name: Report on test fail + uses: actions/upload-artifact@v4 + if: ${{ failure() }} + with: + name: make-testlog + path: | + src/regress_install + src/regress_install.log + src/regression.diffs + src/regression.out + src/results + src/t/results + src/tmp_check + retention-days: 3 diff --git a/ci_scripts/make-build.sh b/ci_scripts/make-build.sh new file mode 100755 index 000000000..b0fa0f467 --- /dev/null +++ b/ci_scripts/make-build.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" +PG_CONFIG="$SCRIPT_DIR/../../pginst/bin/pg_config" + +cd "$SCRIPT_DIR/.." + +export CFLAGS="-Werror -O2" +make PG_CONFIG="$PG_CONFIG" install -j diff --git a/ci_scripts/make-test.sh b/ci_scripts/make-test.sh new file mode 100755 index 000000000..463644607 --- /dev/null +++ b/ci_scripts/make-test.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" + +cd "$SCRIPT_DIR/.." + +OPTS='--set shared_preload_libraries=pg_tde' + +../pginst/bin/pg_ctl -D regress_install -l regress_install.log init -o "$OPTS" + +../pginst/bin/pg_ctl -D regress_install -l regress_install.log start + +make PG_CONFIG=../pginst/bin/pg_config installcheck + +../pginst/bin/pg_ctl -D regress_install stop From be089de5b788caafeb2bf8891761b1d94524bc9f Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Mon, 16 Mar 2026 12:14:54 +0100 Subject: [PATCH 8/9] PG-2029 Do not write TAP output files to source dir Now that we are using Meson it is even more annoying that some of our TAP tests write to t/results. And to make it more useful we make sure to output absolute paths on test failure so people easily can use diff. --- .github/workflows/build-and-test.yml | 1 - .github/workflows/coverage.yml | 1 - .github/workflows/make.yml | 1 - .github/workflows/sanitizers.yml | 1 - .gitignore | 1 - t/pgtde.pm | 13 ++++++++++--- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7d4cba977..43b0ae45e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -119,7 +119,6 @@ jobs: build/regression.diffs build/results build/testrun - src/t/results retention-days: 3 test-psp-with-tde: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a92406fff..0b7f5af9a 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -83,5 +83,4 @@ jobs: build/regression.diffs build/results build/testrun - src/t/results retention-days: 3 diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml index c483a7277..d054e9a36 100644 --- a/.github/workflows/make.yml +++ b/.github/workflows/make.yml @@ -63,6 +63,5 @@ jobs: src/regression.diffs src/regression.out src/results - src/t/results src/tmp_check retention-days: 3 diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 96e402c69..e8e7c173c 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -72,5 +72,4 @@ jobs: build/regression.diffs build/results build/testrun - src/t/results retention-days: 3 diff --git a/.gitignore b/.gitignore index 120a45e96..f9db711fd 100644 --- a/.gitignore +++ b/.gitignore @@ -48,7 +48,6 @@ __pycache__ # Generated subdirectories /log/ /results/ -/t/results/ /tmp_check/ /regress_install/ /regress_install.log diff --git a/t/pgtde.pm b/t/pgtde.pm index d31713d34..de25fef8d 100644 --- a/t/pgtde.pm +++ b/t/pgtde.pm @@ -3,6 +3,7 @@ package PGTDE; use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; +use Cwd qw(abs_path); use File::Basename; use File::Compare; use Test::More; @@ -18,7 +19,12 @@ our $out_filename_with_path; our $debug_out_filename_with_path; my $expected_folder = "t/expected"; -my $results_folder = "t/results"; +my $results_folder; + +INIT +{ + $results_folder = "$PostgreSQL::Test::Utils::tmp_check/results"; +} sub psql { @@ -95,8 +101,9 @@ sub setup_files_dir my ($test_name) = $test_filename =~ /([^.]*)/; - $expected_filename_with_path = "${expected_folder}/${test_name}.out"; - $out_filename_with_path = "${results_folder}/${test_name}.out"; + $expected_filename_with_path = + abs_path("${expected_folder}/${test_name}.out"); + $out_filename_with_path = abs_path("${results_folder}/${test_name}.out"); $debug_out_filename_with_path = "${results_folder}/${test_name}.out.debug"; From 4b74f8949300144f73e6b5d40db109495387978e Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Thu, 2 Apr 2026 10:49:09 +0200 Subject: [PATCH 9/9] PG-2277 Quickly correct the worst parts of CONTRIBUTING.md The instructions are still wrong after this but I did some quick corrections to even be able to update it for adding Meosn support. --- CONTRIBUTING.md | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b23656e3c..70db3f9aa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,39 +68,36 @@ To build `pg_tde` from source code, you require the following: * git * make -* gcc -* pg_config +* gcc or clang +* Percona Server for PostgreSQL 17 or later Refer to the [Building from source code](https://github.com/percona/pg_tde?tab=readme-ov-file#building-from-sources-for-community-postgresql) section for guidelines. ### Run tests -When you work, you should periodically run tests to check that your changes don’t break existing code. - -You can find the tests in the `sql` directory. +You can find the tests in the `sql` and `t` directories. #### Run manually 1. Change the directory to `pg_tde` -**NOTE**: Make sure `postgres` user is the owner of the `pg_tde` directory +2. Build and install `pg_tde` with the following command: -2. Start the tests - 1. If you built PostgreSQL from PGDG, use the following command: + ```sh + make PG_CONFIG=/path/to/postgresql/bin/pg_config install + ``` - ```sh - make installcheck - ``` +3. Start Percona Server for PostgreSQL - 2. If you installed PostgreSQL server from Percona Distribution for PostgreSQL, use the following command: +4. Run the tests using the following command: - ```sh - sudo su postgres bash -c 'make installcheck USE_PGXS=1' - ``` + ```sh + make PG_CONFIG=/path/to/postgresql/bin/pg_config installcheck + ``` #### Run automatically -The tests are run automatically with GitHub actions once you commit and push your changes. Make sure all tests are successfully passed before you proceed. +The tests are run automatically with GitHub actions once you create a pull request. ## Documentation contribution