Fix yezzey address

.github/workflows/build-cloudberry.yml

@@ -271,6 +271,10 @@ jobs:
                 },
                 "enable_core_check":false
               },
+              {"test":"gpcontrib-yagp-hooks-collector",
+               "make_configs":["gpcontrib/yagp_hooks_collector:installcheck"],
+               "extension":"yagp_hooks_collector"
+              },
               {"test":"ic-expandshrink",
                "make_configs":["src/test/isolation2:installcheck-expandshrink"]
               },
@@ -530,10 +534,11 @@ jobs:
         if: needs.check-skip.outputs.should_skip != 'true'
         env:
           SRC_DIR: ${{ github.workspace }}
+          CONFIGURE_EXTRA_OPTS: --with-yagp-hooks-collector
         run: |
           set -eo pipefail
           chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh
-          if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then
+          if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then
             echo "::error::Configure script failed"
             exit 1
           fi
@@ -1390,6 +1395,7 @@ jobs:
         if: success() && needs.check-skip.outputs.should_skip != 'true'
         env:
           SRC_DIR: ${{ github.workspace }}
+          BUILD_DESTINATION: /usr/local/cloudberry-db
         shell: bash {0}
         run: |
           set -o pipefail
@@ -1419,6 +1425,30 @@ jobs:
             PG_OPTS="$PG_OPTS -c optimizer=${{ matrix.pg_settings.optimizer }}"
           fi
 
+          # Create extension if required
+          if [[ "${{ matrix.extension != '' }}" == "true" ]]; then
+            case "${{ matrix.extension }}" in
+              yagp_hooks_collector)
+                if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \
+                  source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \
+                  gpconfig -c shared_preload_libraries -v 'yagp_hooks_collector' && \
+                  gpstop -ra && \
+                  echo 'CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; \
+                        SHOW shared_preload_libraries; \
+                        TABLE pg_extension;' | \
+                    psql postgres"
+                then
+                    echo "Error creating yagp_hooks_collector extension"
+                    exit 1
+                fi
+                ;;
+              *)
+                echo "Unknown extension: ${{ matrix.extension }}"
+                exit 1
+                ;;
+            esac
+          fi
+
           if [[ "${{ matrix.pg_settings.default_table_access_method != '' }}" == "true" ]]; then
             PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}"
           fi

.github/workflows/yezzey-test.yml

@@ -0,0 +1,21 @@
+name: Yezzey testing
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+
+  build_and_run_yezzey:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Test Yezzey
+        run: docker compose -f docker/yezzey/docker-compose.yaml run --build --remove-orphans yezzey
+
+

.gitmodules

@@ -15,3 +15,6 @@
 	path = dependency/yyjson
 	url = https://github.com/ibireme/yyjson.git
 
+[submodule "gpcontrib/yezzey"]
+	path = gpcontrib/yezzey
+	url = https://github.com/open-gpdb/yezzey.git

configure

@@ -722,6 +722,7 @@ with_apr_config
 with_libcurl
 with_rt
 with_zstd
+with_yagp_hooks_collector
 with_libbz2
 LZ4_LIBS
 LZ4_CFLAGS
@@ -942,6 +943,7 @@ with_zlib
 with_lz4
 with_libbz2
 with_zstd
+with_yagp_hooks_collector
 with_rt
 with_libcurl
 with_apr_config
@@ -11150,6 +11152,32 @@ $as_echo "yes" >&6; }
 fi
 fi
 
+#
+# yagp_hooks_collector
+#
+
+
+
+# Check whether --with-yagp-hooks-collector was given.
+if test "${with_yagp_hooks_collector+set}" = set; then :
+  withval=$with_yagp_hooks_collector;
+  case $withval in
+    yes)
+      :
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-yagp-hooks-collector option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_yagp_hooks_collector=no
+
+fi
+
 #
 # Realtime library
 #

configure.ac

@@ -1365,6 +1365,13 @@ PGAC_ARG_BOOL(with, zstd, yes, [do not build with Zstandard],
 AC_MSG_RESULT([$with_zstd])
 AC_SUBST(with_zstd)
 
+#
+# yagp_hooks_collector
+#
+PGAC_ARG_BOOL(with, yagp_hooks_collector, no,
+              [build with YAGP hooks collector extension])
+AC_SUBST(with_yagp_hooks_collector)
+
 if test "$with_zstd" = yes; then
   dnl zstd_errors.h was renamed from error_public.h in v1.4.0
   PKG_CHECK_MODULES([ZSTD], [libzstd >= 1.4.0])

devops/build/automation/cloudberry/scripts/configure-cloudberry.sh

@@ -53,6 +53,7 @@
 #
 # Optional Environment Variables:
 #   LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs)
+#   CONFIGURE_EXTRA_OPTS - Args to pass to configure command
 #   ENABLE_DEBUG - Enable debug build options (true/false, defaults to
 #                  false)
 #
@@ -177,7 +178,8 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \
             --with-uuid=e2fs \
             ${CONFIGURE_MDBLOCALES_OPTS} \
             --with-includes=/usr/local/xerces-c/include \
-            --with-libraries=${BUILD_DESTINATION}/lib || exit 4
+            --with-libraries=${BUILD_DESTINATION}/lib \
+            ${CONFIGURE_EXTRA_OPTS:-""} || exit 4
 log_section_end "Configure"
 
 # Capture version information

docker/yezzey/.dockerignore

@@ -0,0 +1,2 @@
+Dockerfile
+docker-compose.yaml

docker/yezzey/Dockerfile

@@ -0,0 +1,131 @@
+FROM ubuntu:focal
+
+ARG accessKeyId
+ARG secretAccessKey
+ARG bucketName
+ARG s3endpoint
+ARG yezzeyRef
+
+ENV YEZZEY_REF=${yezzeyRef:-v1.8_opengpdb}
+
+ENV AWS_ACCESS_KEY_ID=${accessKeyId}
+ENV AWS_SECRET_ACCESS_KEY=${secretAccessKey}
+ENV S3_BUCKET=${bucketName}
+ENV WALG_S3_PREFIX=s3://${bucketName}/yezzey-test-files
+ENV S3_ENDPOINT=${s3endpoint}
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN useradd -rm -d /home/gpadmin -s /bin/bash -g root -G sudo -u 1001 gpadmin
+
+RUN ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime && echo Europe/London > /etc/timezone \
+&& apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install -y --no-install-recommends --allow-unauthenticated \
+  build-essential libssl-dev gnupg devscripts \
+  openssl libssl-dev debhelper debootstrap \
+  make equivs bison ca-certificates-java ca-certificates \
+  cmake curl cgroup-tools flex gcc-8 g++-8 g++-8-multilib \
+  git krb5-multidev libapr1-dev libbz2-dev libcurl4-gnutls-dev \
+  libevent-dev libkrb5-dev libldap2-dev libperl-dev libreadline6-dev \
+  libssl-dev libxml2-dev libyaml-dev libzstd-dev libaprutil1-dev \
+  libpam0g-dev libpam0g libcgroup1 libyaml-0-2 libldap-2.4-2 libssl1.1 \
+  ninja-build python-dev python-setuptools quilt unzip wget zlib1g-dev libuv1-dev \
+  libgpgme-dev libgpgme11 sudo iproute2 less software-properties-common \
+  openssh-client openssh-server
+
+COPY yezzey_test/install_yproxy.sh /home/gpadmin
+
+RUN ["/home/gpadmin/install_yproxy.sh"]
+
+RUN apt-get install -y locales \
+&& locale-gen "en_US.UTF-8" \
+&& update-locale LC_ALL="en_US.UTF-8"
+
+RUN echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers
+
+USER gpadmin
+WORKDIR /home/gpadmin
+
+COPY yezzey_test/import_gpg_keys.sh /home/gpadmin/
+COPY yezzey_test/priv.gpg /home/gpadmin/yezzey_test/priv.gpg
+COPY yezzey_test/pub.gpg /home/gpadmin/yezzey_test/pub.gpg
+
+RUN ["/home/gpadmin/import_gpg_keys.sh"]
+
+COPY yezzey_test/generate_ssh_key.sh /home/gpadmin/
+
+RUN ["/home/gpadmin/generate_ssh_key.sh"]
+
+
+RUN cd /tmp/ \
+&& git clone https://github.com/boundary/sigar.git \
+&& cd ./sigar/ \
+&& mkdir build && cd build && cmake .. && make \
+&& sudo make install
+
+COPY . /home/gpadmin
+
+RUN sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
+  bison \
+  ccache \
+  cmake \
+  curl \
+  flex \
+  git-core \
+  gcc \
+  g++ \
+  inetutils-ping \
+  krb5-kdc \
+  krb5-admin-server \
+  libapr1-dev \
+  libbz2-dev \
+  libcurl4-gnutls-dev \
+  libevent-dev \
+  libkrb5-dev \
+  libpam-dev \
+  libperl-dev \
+  libreadline-dev \
+  libssl-dev \
+  libxml2-dev \
+  libyaml-dev \
+  libzstd-dev \
+  locales \
+  net-tools \
+  ninja-build \
+  openssh-client \
+  openssh-server \
+  openssl \
+  python3-dev \
+  python3-pip \
+  python3-psutil \
+  python3-pygresql \
+  python-yaml \
+  zlib1g-dev \
+  rsync \
+&& sudo apt install -y libhyperic-sigar-java libaprutil1-dev libuv1-dev
+
+RUN sudo mkdir /usr/local/gpdb \
+&& sudo chown gpadmin:root /usr/local/gpdb
+
+RUN sudo chown -R gpadmin:root /home/gpadmin \
+&& git status
+
+RUN  git submodule update --init 
+RUN rm -fr gpcontrib/yezzey 
+
+# Fetch latest yezzey version
+RUN git clone https://github.com/open-gpdb/yezzey.git gpcontrib/yezzey && cd gpcontrib/yezzey && git fetch origin $YEZZEY_REF:test_branch && git checkout test_branch && cd /home/gpadmin 
+RUN sed -i '/^trusted/d' gpcontrib/yezzey/yezzey.control 
+RUN ./configure --with-perl --with-python --with-libxml --disable-orca --prefix=/usr/local/gpdb \
+--enable-depend --enable-cassert --enable-debug --without-mdblocales --without-zstd CFLAGS='-fno-omit-frame-pointer -Wno-implicit-fallthrough -O3 -pthread' 
+RUN make -j8 && make -j8 install && make -C gpcontrib/yezzey -j8 install
+
+
+RUN echo ${s3endpoint}
+
+RUN sed -i "s/\$AWS_ACCESS_KEY_ID/${accessKeyId}/g" yezzey_test/yproxy.conf \
+&& sed -i "s/\$AWS_SECRET_ACCESS_KEY/${secretAccessKey}/g" yezzey_test/yproxy.conf \
+&& sed -i "s/\$AWS_ENDPOINT/${s3endpoint}/g" yezzey_test/yproxy.conf \
+&& sed -i "s/\$WALG_S3_PREFIX/${bucketName}\/yezzey-test-files/g" yezzey_test/yproxy.conf && cp yezzey_test/yproxy.conf /tmp/yproxy.yaml
+
+ENTRYPOINT ["./yezzey_test/run_tests.sh"]

docker/yezzey/docker-compose.yaml

@@ -0,0 +1,45 @@
+services:
+  minio:
+    image: quay.io/minio/minio
+    command: server --console-address ":9001" /data
+    expose:
+      - "9000"
+      - "9001"
+    environment:
+      MINIO_ROOT_USER: some_key
+      MINIO_ROOT_PASSWORD: some_key
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    hostname: minio
+
+  setup-minio:
+    image: quay.io/minio/mc
+    depends_on:
+      minio:
+        condition: service_healthy
+    entrypoint: |
+      /bin/sh -c "
+      /usr/bin/mc alias set myminio http://minio:9000 some_key some_key
+      /usr/bin/mc mb myminio/gpyezzey
+      /usr/bin/mc mb myminio/gpyezzey2
+      /usr/bin/mc mb myminio/gpyezzey3
+      "
+
+  yezzey:
+    image: yezzey
+    build:
+      context: ../..
+      dockerfile: docker/yezzey/Dockerfile
+      args:
+        accessKeyId: some_key
+        secretAccessKey: some_key
+        bucketName: gpyezzey
+        s3endpoint: "http:\\/\\/minio:9000"
+    depends_on:
+      minio:
+        condition: service_healthy
+      setup-minio:
+        condition: service_completed_successfully

gpcontrib/Makefile

@@ -33,6 +33,9 @@ else
                pg_hint_plan
 endif
 
+ifeq "$(with_yagp_hooks_collector)" "yes"
+	recurse_targets += yagp_hooks_collector
+endif
 ifeq "$(with_zstd)" "yes"
 	recurse_targets += zstd
 endif

gpcontrib/yagp_hooks_collector/.clang-format

@@ -0,0 +1,2 @@
+BasedOnStyle: LLVM
+SortIncludes: false

gpcontrib/yagp_hooks_collector/.gitignore

@@ -0,0 +1,5 @@
+*.o
+*.so
+src/protos/
+.vscode
+compile_commands.json