build and cache grpc on macos, as on linux

.github/workflows/onpush-build-macos.yml

@@ -5,11 +5,39 @@ on:
   push:
 
 env:
-  MACOS_BREW_PACKAGES: "autoconf automake boost cmake gmp grpc openssl pkg-config protobuf libomp libtool"
+  MACOS_BREW_PACKAGES_FOR_GRPC: "autoconf automake cmake openssl pkg-config libtool"
+  MACOS_BREW_PACKAGES: "autoconf automake boost cmake gmp openssl pkg-config libomp libtool"
   HOMEBREW_NO_AUTO_UPDATE: 1
+  PKG_CONFIG_PATH: "/usr/local/opt/openssl/lib/LIBRARY_PATH="
+  LIBRARY_PATH: /usr/local/lib:/usr/local/opt/openssl/lib
+  CC: clang
+  CXX: clang
+  CXXFLAGS: "-stdlib=libc++"
+  LDFLAGS: "-stdlib=libc++"
 
 jobs:
 
+  # Job to build and cache the grpc libraries. The grpc build directory is
+  # populated and cached so that all other jobs can mark this job as a
+  # prerequisite and just run `make install`.
+  #
+  # NOTE: the version number here (in `key` and in script arguments) must be
+  # kept in sync with the key used by the jobs.
+  build-grpc:
+    runs-on: macos-11
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.44.x-test-${{ runner.os }}
+        path: depends/grpc
+    - name: Build grpc
+      run: |
+        brew install ${MACOS_BREW_PACKAGES}
+        brew link llvm@14
+        if ! [ -d depends/grpc ] ; then scripts/install-grpc /usr/local v1.44.x ; fi
+
   # Extract the commits of submodules for use by cache steps
   submodules:
     runs-on: ubuntu-20.04
@@ -26,7 +54,7 @@ jobs:
   # Main build
   build-macos:
     runs-on: macos-11
-    needs: submodules
+    needs: [build-grpc, submodules]
     strategy:
       matrix:
         config: [ Debug, Release ]
@@ -38,18 +66,21 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '3.8'
-    - name: Cache ccache
+    - name: Cache grpc
       uses: actions/cache@v2
       with:
-        key: submodules-${{ needs.submodules.outputs.commits }}-${{ runner.os }}-${{ matrix.config }}
-        path: ~/Library/Caches/ccache
+        key: grpc-1.44.x-test-${{ runner.os }}
+        path: depends/grpc
     - name: Cache pip (for mpc tests)
       uses: actions/cache@v2
       with:
         path: ~/Library/Caches/pip
         key: build-macos-pip-${{ hashFiles('**/setup.py') }}-${{ runner.os }}
     - name: Install Dependencies
       run: |
+    - name: Install dependencies
+      run: |
+        INSTALL_ONLY=1 scripts/install-grpc /usr/local v1.44.x
         brew install ${MACOS_BREW_PACKAGES}
         brew link llvm@14
         which clang

.github/workflows/onpush-build-ubuntu.yml

@@ -51,7 +51,6 @@ jobs:
     - name: Install dependencies
       run: |
         INSTALL_ONLY=1 scripts/install-grpc /usr v1.44.x
-        sudo apt install -y ccache
     - name: Execute
       run: CI_CHECK_FORMAT=1 CI_MPC_TESTS=1 CI_CONFIG=${{ matrix.config }} scripts/ci build
 
@@ -74,7 +73,6 @@ jobs:
     - name: Install dependencies
       run: |
         INSTALL_ONLY=1 scripts/install-grpc /usr v1.44.x
-        sudo apt install -y ccache
     - name: Execute
       run: CI_CONFIG=Release CI_ZKSNARK=PGHR13 scripts/ci build
 
@@ -97,6 +95,5 @@ jobs:
     - name: Install dependencies
       run: |
         INSTALL_ONLY=1 scripts/install-grpc /usr v1.44.x
-        sudo apt install -y ccache
     - name: Execute
       run: CI_CONFIG=Release CI_CURVE=BLS12_377 scripts/ci build

scripts/build_utils.sh

@@ -47,5 +47,13 @@ function cpp_build_setup() {
                  libxslt1-dev \
                  pkg-config
         fi
+
+    elif [ "${platform}" == "Darwin" ] ; then
+        # openssl_path=$(brew --prefix openssl)
+        # export PKG_CONFIG_PATH="${openssl_path}/lib/pkgconfig"
+        # export LIBRARY_PATH="${openssl_path}/lib"
+        export LDFLAGS="-L/usr/local/lib -L${openssl_path}/lib"
+        export CPPFLAGS="-I/usr/local/include -I${openssl_path}/include"
+        export CXXFLAGS="-I/usr/local/include -I${openssl_path}/include"
     fi
 }

scripts/ci

@@ -162,11 +162,7 @@ function build() {
     cxx_flags="-Werror"
 
     if [ "${platform}" == "Darwin" ] ; then
-        openssl_path=$(brew --prefix openssl)
-        export PKG_CONFIG_PATH="${openssl_path}/lib/pkgconfig"
-        export LIBRARY_PATH="${openssl_path}/lib"
-        export LDFLAGS="-L/usr/local/lib -L${openssl_path}/lib"
-        export CPPFLAGS="-I/usr/local/include -I${openssl_path}/include"
+        # openssl_path is set by cpp_build_setup in build_utils.
 
         cxx_flags="${cxx_flags} -I${openssl_path}/include"
         cxx_flags="${cxx_flags} -Wno-deprecated-declarations"

scripts/install-grpc

@@ -25,11 +25,23 @@ if [ "${INSTALL_ONLY}" != "1" ] ; then
     cpp_build_setup
 fi
 
+
+if [ ${platform} == "Darwin" ] ; then
+    function nproc() {
+        sysctl -n hw.ncpu
+    }
+fi
+
+
 pushd depends
 
     if ! [ -d grpc ] ; then
-        git clone --depth 1 -b ${VERSION} https://github.com/grpc/grpc
-        git -C grpc submodule update --depth 1 --init --recursive
+        git clone \
+            --depth 1 \
+            --shallow-submodules \
+            --recurse-submodules \
+            -b ${VERSION} \
+            https://github.com/grpc/grpc
     else
         echo grpc directory already exists. skipping download.
     fi
@@ -47,26 +59,27 @@ pushd depends
         popd # third_party/protobuf
 
         # Install grpc
-        mkdir -p build
-        pushd build
-            if [ "${INSTALL_ONLY}" != "1" ] ; then
-                cmake \
-                    -DCMAKE_PREFIX_PATH=${INSTALL_DIR}       \
-                    -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}    \
-                    -DCMAKE_BUILD_TYPE=Release               \
-                    -DgRPC_INSTALL=ON                        \
-                    -DgRPC_BUILD_TESTS=OFF                   \
-                    -DBUILD_TESTING=OFF                      \
-                    -DgRPC_SSL_PROVIDER=package              \
-                    -DgRPC_PROTOBUF_PROVIDER=package         \
-                    -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF      \
-                    -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF        \
-                    -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF \
-                    -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF         \
-                    -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF        \
-                    ..
-                make -j"$(($(nproc)+1))"
-            fi
+        mkdir -p build-grpc
+        pushd build-grpc
+            # if [ "${INSTALL_ONLY}" != "1" ] ; then
+            cmake \
+                -DCMAKE_PREFIX_PATH=${INSTALL_DIR}       \
+                -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}    \
+                -DCMAKE_BUILD_TYPE=Release               \
+                -DgRPC_INSTALL=ON                        \
+                -DgRPC_BUILD_TESTS=OFF                   \
+                -DBUILD_TESTING=OFF                      \
+                -DgRPC_SSL_PROVIDER=package              \
+                -DgRPC_PROTOBUF_PROVIDER=package         \
+                -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF      \
+                -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF        \
+                -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF \
+                -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF         \
+                -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF        \
+                ..
+
+            make -j $(($(nproc)+1))
+            # fi
             sudo make install
         popd # build