From 9e1008f8ff9a2c7fc56e1bbcf0b88c6d48c65b9f Mon Sep 17 00:00:00 2001 From: Dennis Kovalenko Date: Wed, 2 Aug 2023 09:56:08 +0400 Subject: [PATCH 01/14] ADBDEV-3774 Use custom allocators for zstd (#566) gpdb uses zstd to compress workfiles, but it can't control memory allocation within zstd. Meanwhile, zstd allocates as much memory as it needs which results in ambiguous error message in a case of memory exhaustion and can make OOM killer stop gpdb processes with force. This patch forces zstd to use our custom allocator which can keep track of allocated memory. Since this zstd feature is experimental and its api can change, we also had to link zstd statically. --- configure | 46 +++++++++++++++--------------- configure.in | 2 +- src/backend/storage/file/buffile.c | 26 +++++++++++++++-- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/configure b/configure index 5cb43402954..b00efb3703a 100755 --- a/configure +++ b/configure @@ -11601,50 +11601,50 @@ fi fi if test "$with_zstd" = yes; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ZSTD_compressCCtx in -lzstd" >&5 -$as_echo_n "checking for ZSTD_compressCCtx in -lzstd... " >&6; } -if ${ac_cv_lib_zstd_ZSTD_compressCCtx+:} false; then : - $as_echo_n "(cached) " >&6 -else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ZSTD_compressCCtx in -l:libzstd.a" >&5 +printf %s "checking for ZSTD_compressCCtx in -l:libzstd.a... " >&6; } +if test ${ac_cv_lib__libzstd_a_ZSTD_compressCCtx+y} +then : + printf %s "(cached) " >&6 +else $as_nop ac_check_lib_save_LIBS=$LIBS -LIBS="-lzstd $LIBS" +LIBS="-l:libzstd.a $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif char ZSTD_compressCCtx (); int -main () +main (void) { return ZSTD_compressCCtx (); ; return 0; } _ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_zstd_ZSTD_compressCCtx=yes -else - ac_cv_lib_zstd_ZSTD_compressCCtx=no +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib__libzstd_a_ZSTD_compressCCtx=yes +else $as_nop + ac_cv_lib__libzstd_a_ZSTD_compressCCtx=no fi -rm -f core conftest.err conftest.$ac_objext \ +rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_ZSTD_compressCCtx" >&5 -$as_echo "$ac_cv_lib_zstd_ZSTD_compressCCtx" >&6; } -if test "x$ac_cv_lib_zstd_ZSTD_compressCCtx" = xyes; then : - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBZSTD 1 -_ACEOF +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib__libzstd_a_ZSTD_compressCCtx" >&5 +printf "%s\n" "$ac_cv_lib__libzstd_a_ZSTD_compressCCtx" >&6; } +if test "x$ac_cv_lib__libzstd_a_ZSTD_compressCCtx" = xyes +then : - LIBS="-lzstd $LIBS" + LIBS="-l:libzstd.a $LIBS" -else +printf "%s\n" "#define HAVE_LIBZSTD 1" >>confdefs.h + + +else $as_nop as_fn_error $? "zstd library not found If you have libzstd already installed, see config.log for details on the failure. It is possible the compiler isn't looking in the proper directory. diff --git a/configure.in b/configure.in index 2db78a53ec0..1efec070341 100644 --- a/configure.in +++ b/configure.in @@ -1428,7 +1428,7 @@ Use --without-zlib to disable zlib support.])]) fi if test "$with_zstd" = yes; then - AC_CHECK_LIB(zstd, ZSTD_compressCCtx, [], + AC_CHECK_LIB(:libzstd.a, ZSTD_compressCCtx, [], [AC_MSG_ERROR([zstd library not found If you have libzstd already installed, see config.log for details on the failure. It is possible the compiler isn't looking in the proper directory. diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index 02ae0c4f352..204bb55ad53 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -60,6 +60,7 @@ #include "postgres.h" #ifdef HAVE_LIBZSTD +#define ZSTD_STATIC_LINKING_ONLY #include #endif @@ -72,6 +73,7 @@ #include "cdb/cdbvars.h" #include "storage/gp_compress.h" +#include "utils/gp_alloc.h" #include "utils/faultinjector.h" #include "utils/memutils.h" #include "utils/workfile_mgr.h" @@ -998,6 +1000,18 @@ BufFilePledgeSequential(BufFile *buffile) #define BUFFILE_ZSTD_COMPRESSION_LEVEL 1 +void * +customAlloc(void *opaque, size_t size) +{ + return MemoryContextAlloc(TopMemoryContext, size); +} + +void +customFree(void *opaque, void *address) +{ + pfree(address); +} + /* * Temporary buffer used during compresion. It's used only within the * functions, so we can allocate this once and reuse it for all files. @@ -1012,6 +1026,10 @@ BufFileStartCompression(BufFile *file) { ResourceOwner oldowner; size_t ret; + ZSTD_customMem customMem; + + customMem.customAlloc = customAlloc; + customMem.customFree = customFree; /* * When working with compressed files, we rely on libzstd's buffer, @@ -1038,7 +1056,7 @@ BufFileStartCompression(BufFile *file) CurrentResourceOwner = file->resowner; file->zstd_context = zstd_alloc_context(); - file->zstd_context->cctx = ZSTD_createCStream(); + file->zstd_context->cctx = ZSTD_createCStream_advanced(customMem); if (!file->zstd_context->cctx) elog(ERROR, "out of memory"); ret = ZSTD_initCStream(file->zstd_context->cctx, BUFFILE_ZSTD_COMPRESSION_LEVEL); @@ -1097,6 +1115,10 @@ BufFileEndCompression(BufFile *file) ZSTD_outBuffer output; size_t ret; int wrote; + ZSTD_customMem customMem; + + customMem.customAlloc = customAlloc; + customMem.customFree = customFree; Assert(file->state == BFS_COMPRESSED_WRITING); @@ -1122,7 +1144,7 @@ BufFileEndCompression(BufFile *file) file->uncompressed_bytes, file->maxoffset); /* Done writing. Initialize for reading */ - file->zstd_context->dctx = ZSTD_createDStream(); + file->zstd_context->dctx = ZSTD_createDStream_advanced(customMem); if (!file->zstd_context->dctx) elog(ERROR, "out of memory"); ret = ZSTD_initDStream(file->zstd_context->dctx); From be0e1c72149bd19f8376ea500e6ebc24d6944a49 Mon Sep 17 00:00:00 2001 From: Victor Kremensky Date: Thu, 13 Mar 2025 11:45:42 +0000 Subject: [PATCH 02/14] Add GUC --- src/backend/storage/file/buffile.c | 14 ++++++++++++-- src/backend/utils/misc/guc_gp.c | 14 +++++++++++++- src/include/utils/guc.h | 1 + src/include/utils/sync_guc_name.h | 1 + 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index 204bb55ad53..e33d5ec152d 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -1056,7 +1056,13 @@ BufFileStartCompression(BufFile *file) CurrentResourceOwner = file->resowner; file->zstd_context = zstd_alloc_context(); - file->zstd_context->cctx = ZSTD_createCStream_advanced(customMem); + if (gp_correct_zstd_memory_counting) + { + file->zstd_context->cctx = ZSTD_createCStream_advanced(customMem); + }else{ + file->zstd_context->cctx = ZSTD_createCStream(); + } + if (!file->zstd_context->cctx) elog(ERROR, "out of memory"); ret = ZSTD_initCStream(file->zstd_context->cctx, BUFFILE_ZSTD_COMPRESSION_LEVEL); @@ -1144,7 +1150,11 @@ BufFileEndCompression(BufFile *file) file->uncompressed_bytes, file->maxoffset); /* Done writing. Initialize for reading */ - file->zstd_context->dctx = ZSTD_createDStream_advanced(customMem); + if (gp_correct_zstd_memory_counting){ + file->zstd_context->dctx = ZSTD_createDStream_advanced(customMem); + }else{ + file->zstd_context->dctx = ZSTD_createDStream(); + } if (!file->zstd_context->dctx) elog(ERROR, "out of memory"); ret = ZSTD_initDStream(file->zstd_context->dctx); diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index 046cbf5782d..c6914809640 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -159,7 +159,7 @@ bool gp_enable_exchange_default_partition = false; int dtx_phase2_retry_count = 0; bool gp_log_suboverflow_statement = false; bool gp_use_synchronize_seqscans_catalog_vacuum_full = false; - +bool gp_correct_zstd_memory_counting = true; bool log_dispatch_stats = false; int explain_memory_verbosity = 0; @@ -3294,10 +3294,22 @@ struct config_bool ConfigureNamesBool_gp[] = NULL }, &gp_use_synchronize_seqscans_catalog_vacuum_full, + true, + NULL, NULL, NULL + }, + + { + {"gp_correct_zstd_memory_counting", PGC_USERSET, COMPAT_OPTIONS_PREVIOUS, + gettext_noop("Enables normal memory counting in zstd (not using malloc)"), + NULL + }, + &gp_correct_zstd_memory_counting, false, NULL, NULL, NULL }, + + { {"gp_external_fail_on_eof", PGC_USERSET, EXTERNAL_TABLES, gettext_noop("Fail with ERROR and rollback on 'unexpected end of file' error when used with a CUSTOM formatter."), diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index 31f5fa4d8d6..918282962d4 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -300,6 +300,7 @@ extern bool gp_enable_exchange_default_partition; extern int dtx_phase2_retry_count; extern bool gp_log_suboverflow_statement; extern bool gp_use_synchronize_seqscans_catalog_vacuum_full; +extern bool gp_correct_zstd_memory_counting; /* WAL replication debug gucs */ extern bool debug_walrepl_snd; diff --git a/src/include/utils/sync_guc_name.h b/src/include/utils/sync_guc_name.h index f2ba6027256..95b3807ca10 100644 --- a/src/include/utils/sync_guc_name.h +++ b/src/include/utils/sync_guc_name.h @@ -129,3 +129,4 @@ "gp_resgroup_debug_wait_queue", "gp_log_resqueue_priority_sleep_time", "yc_allow_copy_to_program", + "gp_correct_zstd_memory_counting", From 8bcc9dd57dad540dd72d469df19092c29bc57160 Mon Sep 17 00:00:00 2001 From: Victor Kremensky Date: Thu, 13 Mar 2025 11:52:34 +0000 Subject: [PATCH 03/14] s/gp_correct_zstd_memory_counting/gp_resgroup_enable_zstd_counting --- src/backend/storage/file/buffile.c | 4 ++-- src/backend/utils/misc/guc_gp.c | 6 +++--- src/include/utils/guc.h | 2 +- src/include/utils/sync_guc_name.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index e33d5ec152d..d3742fcb4b4 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -1056,7 +1056,7 @@ BufFileStartCompression(BufFile *file) CurrentResourceOwner = file->resowner; file->zstd_context = zstd_alloc_context(); - if (gp_correct_zstd_memory_counting) + if (gp_resgroup_enable_zstd_counting) { file->zstd_context->cctx = ZSTD_createCStream_advanced(customMem); }else{ @@ -1150,7 +1150,7 @@ BufFileEndCompression(BufFile *file) file->uncompressed_bytes, file->maxoffset); /* Done writing. Initialize for reading */ - if (gp_correct_zstd_memory_counting){ + if (gp_resgroup_enable_zstd_counting){ file->zstd_context->dctx = ZSTD_createDStream_advanced(customMem); }else{ file->zstd_context->dctx = ZSTD_createDStream(); diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index c6914809640..dc80d604ba8 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -159,7 +159,7 @@ bool gp_enable_exchange_default_partition = false; int dtx_phase2_retry_count = 0; bool gp_log_suboverflow_statement = false; bool gp_use_synchronize_seqscans_catalog_vacuum_full = false; -bool gp_correct_zstd_memory_counting = true; +bool gp_resgroup_enable_zstd_counting = true; bool log_dispatch_stats = false; int explain_memory_verbosity = 0; @@ -3299,11 +3299,11 @@ struct config_bool ConfigureNamesBool_gp[] = }, { - {"gp_correct_zstd_memory_counting", PGC_USERSET, COMPAT_OPTIONS_PREVIOUS, + {"gp_resgroup_enable_zstd_counting", PGC_USERSET, COMPAT_OPTIONS_PREVIOUS, gettext_noop("Enables normal memory counting in zstd (not using malloc)"), NULL }, - &gp_correct_zstd_memory_counting, + &gp_resgroup_enable_zstd_counting, false, NULL, NULL, NULL }, diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index 918282962d4..ec0411c3d4e 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -300,7 +300,7 @@ extern bool gp_enable_exchange_default_partition; extern int dtx_phase2_retry_count; extern bool gp_log_suboverflow_statement; extern bool gp_use_synchronize_seqscans_catalog_vacuum_full; -extern bool gp_correct_zstd_memory_counting; +extern bool gp_resgroup_enable_zstd_counting; /* WAL replication debug gucs */ extern bool debug_walrepl_snd; diff --git a/src/include/utils/sync_guc_name.h b/src/include/utils/sync_guc_name.h index 95b3807ca10..5b377784420 100644 --- a/src/include/utils/sync_guc_name.h +++ b/src/include/utils/sync_guc_name.h @@ -129,4 +129,4 @@ "gp_resgroup_debug_wait_queue", "gp_log_resqueue_priority_sleep_time", "yc_allow_copy_to_program", - "gp_correct_zstd_memory_counting", + "gp_resgroup_enable_zstd_counting", From 7f329d80184d74c14d79df1823faf5907f4a4d3e Mon Sep 17 00:00:00 2001 From: Victor Kremensky Date: Thu, 13 Mar 2025 15:40:13 +0000 Subject: [PATCH 04/14] add include --- src/backend/storage/file/buffile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index d3742fcb4b4..201b2e75cf1 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -77,7 +77,7 @@ #include "utils/faultinjector.h" #include "utils/memutils.h" #include "utils/workfile_mgr.h" - +#include "utils/guc.h" /* * This data structure represents a buffered file that consists of one * physical file (accessed through a virtual file descriptor From 1a9f172b2f5baab37add726293366160e3297405 Mon Sep 17 00:00:00 2001 From: Victor Kremensky Date: Fri, 14 Mar 2025 11:20:35 +0000 Subject: [PATCH 05/14] zstd --- docker/test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index b39df13c00f..353e5f9c114 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -27,7 +27,7 @@ RUN ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime && echo Europe/Lond libpam0g-dev libpam0g libcgroup1 libyaml-0-2 libldap2-dev openssl \ ninja-build python2-dev python-setuptools quilt unzip wget zlib1g-dev libuv1-dev \ libgpgme-dev libgpgme11 sudo iproute2 less software-properties-common \ - openssh-client openssh-server + openssh-client openssh-server zstd RUN apt-get install -y locales \ && locale-gen "en_US.UTF-8" \ From 51e88cbc14471cbcc22c495322468a6361a59f97 Mon Sep 17 00:00:00 2001 From: Victor Kremensky Date: Fri, 14 Mar 2025 11:51:59 +0000 Subject: [PATCH 06/14] revert --- docker/test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 353e5f9c114..b39df13c00f 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -27,7 +27,7 @@ RUN ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime && echo Europe/Lond libpam0g-dev libpam0g libcgroup1 libyaml-0-2 libldap2-dev openssl \ ninja-build python2-dev python-setuptools quilt unzip wget zlib1g-dev libuv1-dev \ libgpgme-dev libgpgme11 sudo iproute2 less software-properties-common \ - openssh-client openssh-server zstd + openssh-client openssh-server RUN apt-get install -y locales \ && locale-gen "en_US.UTF-8" \ From f17293ecfcebf7f65a0b45488770b21e87b093a2 Mon Sep 17 00:00:00 2001 From: Victor Kremensky Date: Fri, 14 Mar 2025 11:53:44 +0000 Subject: [PATCH 07/14] we use other package zstd? we need our docker image --- .github/workflows/greenplum-abi-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/greenplum-abi-tests.yml b/.github/workflows/greenplum-abi-tests.yml index b23e79080d8..e3ed867e16b 100644 --- a/.github/workflows/greenplum-abi-tests.yml +++ b/.github/workflows/greenplum-abi-tests.yml @@ -114,7 +114,7 @@ jobs: ./configure --with-quicklz --disable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy \ --enable-orca --with-libxml --with-pythonsrc-ext --with-uuid=e2fs --with-pgport=5432 --enable-tap-tests \ --enable-debug-extensions --with-perl --with-python --with-openssl --with-pam --with-ldap --with-includes="" \ - --without-mdblocales \ + --without-mdblocales --without-zstd\ --with-libraries="" --disable-rpath \ --prefix=/usr/local/greenplum-db-devel \ --mandir=/usr/local/greenplum-db-devel/man From 08f652b9fee0c747ceb7a6d52f0b90e3eb3693d0 Mon Sep 17 00:00:00 2001 From: Victor Kremensky Date: Tue, 25 Mar 2025 14:15:17 +0000 Subject: [PATCH 08/14] rename to gp_enable_zstd_memory_accounting --- docker/test/run_tests.sh | 1 - src/backend/storage/file/buffile.c | 4 ++-- src/backend/utils/misc/guc_gp.c | 6 +++--- src/include/utils/guc.h | 2 +- src/include/utils/sync_guc_name.h | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docker/test/run_tests.sh b/docker/test/run_tests.sh index b34ec14ccd1..e84d2afa0e5 100755 --- a/docker/test/run_tests.sh +++ b/docker/test/run_tests.sh @@ -70,7 +70,6 @@ gpconfig -c shared_preload_libraries -v yezzey if [ "${TEST_CGROUP}" = "true" ]; then gpconfig -c gp_resource_manager -v "group" fi - gpstop -a -i && gpstart -a createdb $USER diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index 201b2e75cf1..75d517a289e 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -1056,7 +1056,7 @@ BufFileStartCompression(BufFile *file) CurrentResourceOwner = file->resowner; file->zstd_context = zstd_alloc_context(); - if (gp_resgroup_enable_zstd_counting) + if (gp_enable_zstd_memory_accounting) { file->zstd_context->cctx = ZSTD_createCStream_advanced(customMem); }else{ @@ -1150,7 +1150,7 @@ BufFileEndCompression(BufFile *file) file->uncompressed_bytes, file->maxoffset); /* Done writing. Initialize for reading */ - if (gp_resgroup_enable_zstd_counting){ + if (gp_enable_zstd_memory_accounting){ file->zstd_context->dctx = ZSTD_createDStream_advanced(customMem); }else{ file->zstd_context->dctx = ZSTD_createDStream(); diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index dc80d604ba8..5203116d76e 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -159,7 +159,7 @@ bool gp_enable_exchange_default_partition = false; int dtx_phase2_retry_count = 0; bool gp_log_suboverflow_statement = false; bool gp_use_synchronize_seqscans_catalog_vacuum_full = false; -bool gp_resgroup_enable_zstd_counting = true; +bool gp_enable_zstd_memory_accounting = true; bool log_dispatch_stats = false; int explain_memory_verbosity = 0; @@ -3299,11 +3299,11 @@ struct config_bool ConfigureNamesBool_gp[] = }, { - {"gp_resgroup_enable_zstd_counting", PGC_USERSET, COMPAT_OPTIONS_PREVIOUS, + {"gp_enable_zstd_memory_accounting", PGC_USERSET, COMPAT_OPTIONS_PREVIOUS, gettext_noop("Enables normal memory counting in zstd (not using malloc)"), NULL }, - &gp_resgroup_enable_zstd_counting, + &gp_enable_zstd_memory_accounting, false, NULL, NULL, NULL }, diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index ec0411c3d4e..c25a49388b2 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -300,7 +300,7 @@ extern bool gp_enable_exchange_default_partition; extern int dtx_phase2_retry_count; extern bool gp_log_suboverflow_statement; extern bool gp_use_synchronize_seqscans_catalog_vacuum_full; -extern bool gp_resgroup_enable_zstd_counting; +extern bool gp_enable_zstd_memory_accounting; /* WAL replication debug gucs */ extern bool debug_walrepl_snd; diff --git a/src/include/utils/sync_guc_name.h b/src/include/utils/sync_guc_name.h index 5b377784420..82a9b3733d2 100644 --- a/src/include/utils/sync_guc_name.h +++ b/src/include/utils/sync_guc_name.h @@ -129,4 +129,4 @@ "gp_resgroup_debug_wait_queue", "gp_log_resqueue_priority_sleep_time", "yc_allow_copy_to_program", - "gp_resgroup_enable_zstd_counting", + "gp_enable_zstd_memory_accounting", From c47748a60d1e76c74eed8fc5e7c4c69a6670118f Mon Sep 17 00:00:00 2001 From: Victor Kremensky Date: Tue, 1 Apr 2025 06:50:09 +0000 Subject: [PATCH 09/14] zstd_memory_context --- src/backend/storage/file/buffile.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index 75d517a289e..d50cdb4ecd2 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -999,11 +999,14 @@ BufFilePledgeSequential(BufFile *buffile) #ifdef HAVE_LIBZSTD #define BUFFILE_ZSTD_COMPRESSION_LEVEL 1 +struct{ + MemoryContext context; +} zstd_memory_context; void * customAlloc(void *opaque, size_t size) { - return MemoryContextAlloc(TopMemoryContext, size); + return MemoryContextAlloc(zstd_memory_context.context, size); } void @@ -1027,6 +1030,7 @@ BufFileStartCompression(BufFile *file) ResourceOwner oldowner; size_t ret; ZSTD_customMem customMem; + zstd_memory_context.context = AllocSetContextCreate(TopMemoryContext,"zstd_context", ALLOCSET_DEFAULT_SIZES); customMem.customAlloc = customAlloc; customMem.customFree = customFree; @@ -1044,7 +1048,7 @@ BufFileStartCompression(BufFile *file) } if (compression_buffer == NULL) - compression_buffer = MemoryContextAlloc(TopMemoryContext, BLCKSZ); + compression_buffer = MemoryContextAlloc(zstd_memory_context.context, BLCKSZ); /* * Make sure the zstd handle is kept in the same resource owner as From 51a3a43601d7ebce4cf3007c8ae46cca909cf3d2 Mon Sep 17 00:00:00 2001 From: Victor Kremensky Date: Tue, 1 Apr 2025 07:24:23 +0000 Subject: [PATCH 10/14] simplify --- src/backend/storage/file/buffile.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index d50cdb4ecd2..e8f2c37ae88 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -999,14 +999,11 @@ BufFilePledgeSequential(BufFile *buffile) #ifdef HAVE_LIBZSTD #define BUFFILE_ZSTD_COMPRESSION_LEVEL 1 -struct{ - MemoryContext context; -} zstd_memory_context; - +MemoryContext zstd_memory_context; void * customAlloc(void *opaque, size_t size) { - return MemoryContextAlloc(zstd_memory_context.context, size); + return MemoryContextAlloc(zstd_memory_context, size); } void @@ -1030,7 +1027,7 @@ BufFileStartCompression(BufFile *file) ResourceOwner oldowner; size_t ret; ZSTD_customMem customMem; - zstd_memory_context.context = AllocSetContextCreate(TopMemoryContext,"zstd_context", ALLOCSET_DEFAULT_SIZES); + zstd_memory_context = AllocSetContextCreate(TopMemoryContext,"zstd_context", ALLOCSET_DEFAULT_SIZES); customMem.customAlloc = customAlloc; customMem.customFree = customFree; @@ -1048,7 +1045,7 @@ BufFileStartCompression(BufFile *file) } if (compression_buffer == NULL) - compression_buffer = MemoryContextAlloc(zstd_memory_context.context, BLCKSZ); + compression_buffer = MemoryContextAlloc(zstd_memory_context, BLCKSZ); /* * Make sure the zstd handle is kept in the same resource owner as From 5f73e2c9af1fe327e120a11d19a2c0a5c02a2070 Mon Sep 17 00:00:00 2001 From: Victor Kremensky Date: Wed, 2 Apr 2025 14:44:26 +0000 Subject: [PATCH 11/14] better --- src/backend/storage/file/buffile.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index e8f2c37ae88..8b71dc41bd1 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -123,6 +123,7 @@ struct BufFile /* ZStandard compression support */ #ifdef HAVE_LIBZSTD + MemoryContext pg_zstd_context; zstd_context *zstd_context; /* ZStandard library handles. */ /* @@ -366,6 +367,8 @@ BufFileClose(BufFile *file) #ifdef HAVE_LIBZSTD if (file->zstd_context) zstd_free_context(file->zstd_context); + if (file->pg_zstd_context) + MemoryContextDelete(file->pg_zstd_context); #endif pfree(file); @@ -999,11 +1002,10 @@ BufFilePledgeSequential(BufFile *buffile) #ifdef HAVE_LIBZSTD #define BUFFILE_ZSTD_COMPRESSION_LEVEL 1 -MemoryContext zstd_memory_context; void * customAlloc(void *opaque, size_t size) { - return MemoryContextAlloc(zstd_memory_context, size); + return MemoryContextAlloc((MemoryContext)opaque, size); } void @@ -1027,11 +1029,13 @@ BufFileStartCompression(BufFile *file) ResourceOwner oldowner; size_t ret; ZSTD_customMem customMem; - zstd_memory_context = AllocSetContextCreate(TopMemoryContext,"zstd_context", ALLOCSET_DEFAULT_SIZES); + #ifdef HAVE_LIBZSTD + file->pg_zstd_context = AllocSetContextCreate(TopMemoryContext,"zstd_context", ALLOCSET_DEFAULT_SIZES); + #endif customMem.customAlloc = customAlloc; customMem.customFree = customFree; - + customMem.opaque = file->pg_zstd_context; /* * When working with compressed files, we rely on libzstd's buffer, * and the BufFile's own buffer is unused. It's a bit silly that we @@ -1045,7 +1049,7 @@ BufFileStartCompression(BufFile *file) } if (compression_buffer == NULL) - compression_buffer = MemoryContextAlloc(zstd_memory_context, BLCKSZ); + compression_buffer = MemoryContextAlloc(file->pg_zstd_context, BLCKSZ); /* * Make sure the zstd handle is kept in the same resource owner as From 59ba458217fd2cce9d59cf7f5c409e3a8f764e1b Mon Sep 17 00:00:00 2001 From: Victor Kremensky Date: Thu, 3 Apr 2025 10:41:26 +0000 Subject: [PATCH 12/14] redurant ifdef --- src/backend/storage/file/buffile.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index 8b71dc41bd1..05a2ecd806e 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -1029,9 +1029,7 @@ BufFileStartCompression(BufFile *file) ResourceOwner oldowner; size_t ret; ZSTD_customMem customMem; - #ifdef HAVE_LIBZSTD file->pg_zstd_context = AllocSetContextCreate(TopMemoryContext,"zstd_context", ALLOCSET_DEFAULT_SIZES); - #endif customMem.customAlloc = customAlloc; customMem.customFree = customFree; From 6fed14653b1c816ba36f89670405ad65c1eb2b06 Mon Sep 17 00:00:00 2001 From: Victor Kremensky Date: Thu, 17 Apr 2025 10:11:20 +0000 Subject: [PATCH 13/14] add context --- src/backend/storage/file/buffile.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index 05a2ecd806e..a87ecefbe82 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -1128,6 +1128,7 @@ BufFileEndCompression(BufFile *file) customMem.customAlloc = customAlloc; customMem.customFree = customFree; + customMem.opaque = file->pg_zstd_context; Assert(file->state == BFS_COMPRESSED_WRITING); From 4e82590fd84e086a6c7064789f75cd3f384e60be Mon Sep 17 00:00:00 2001 From: Victor Kremensky Date: Thu, 17 Apr 2025 10:24:43 +0000 Subject: [PATCH 14/14] fix compression_buffer context --- src/backend/storage/file/buffile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index a87ecefbe82..4d375506910 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -1047,7 +1047,7 @@ BufFileStartCompression(BufFile *file) } if (compression_buffer == NULL) - compression_buffer = MemoryContextAlloc(file->pg_zstd_context, BLCKSZ); + compression_buffer = MemoryContextAlloc(TopMemoryContext, BLCKSZ); /* * Make sure the zstd handle is kept in the same resource owner as