From 6a1c460194dff5246f78abd5c8897318dfc0c860 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Tue, 24 Oct 2023 14:12:26 +0000 Subject: [PATCH 1/9] udgrade pybind dependency to v2.11.1 --- extern/pybind11 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/pybind11 b/extern/pybind11 index 3b1dbeb..8a099e4 160000 --- a/extern/pybind11 +++ b/extern/pybind11 @@ -1 +1 @@ -Subproject commit 3b1dbebabc801c9cf6f0953a4c20b904d444f879 +Subproject commit 8a099e44b3d5f85b20f05828d919d2332a8de841 From e3041102f1d5451376fa628c52a8c8f8563e4331 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Tue, 24 Oct 2023 14:13:27 +0000 Subject: [PATCH 2/9] update to recent gcc version in dockerfile --- Dockerfile.ubuntu | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index ef71230..91e90f7 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -6,7 +6,7 @@ RUN find /lib/x86_64-linux-gnu/ /usr/lib/x86_64-linux-gnu/ -name '*.so*' > /shar RUN apt-get update && \ apt-get install -y --no-install-recommends \ dpkg-dev \ - g++-8 \ + g++-10 \ libc6-dev \ libssl-dev \ openssl \ @@ -15,13 +15,15 @@ RUN apt-get update && \ libboost-system-dev \ libboost-thread-dev \ libgmp-dev \ + python3 \ + libpython3-dev \ libcurl4-openssl-dev \ && rm -rf /var/lib/apt/lists/* -# We only installed gcc-8 -> make default -RUN ln -s /usr/bin/gcc-8 /usr/bin/gcc \ - && ln -s /usr/bin/g++-8 /usr/bin/g++ \ - && ln -s /usr/bin/gcov-8 /usr/bin/gcov +# We only installed gcc-10 -> make default +RUN ln -s /usr/bin/gcc-10 /usr/bin/gcc \ + && ln -s /usr/bin/g++-10 /usr/bin/g++ \ + && ln -s /usr/bin/gcov-10 /usr/bin/gcov WORKDIR /app COPY extern extern @@ -29,6 +31,7 @@ COPY cmake cmake COPY include include COPY test test COPY scripts scripts +COPY pysel pysel COPY CMakeLists.txt sepilinker.cpp ./ # build! From 7963ae1daab15c2bb55f3deb35de42281075c5d5 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Tue, 16 Jul 2024 07:46:45 +0000 Subject: [PATCH 3/9] :construction_worker: Add python3 to dockerfile --- Dockerfile.distroless | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile.distroless b/Dockerfile.distroless index 3fc84f5..a398a85 100644 --- a/Dockerfile.distroless +++ b/Dockerfile.distroless @@ -19,6 +19,9 @@ RUN apt-get update && \ libboost-thread-dev \ libgmp-dev \ libcurl4-openssl-dev \ + libpython3-dev \ + python3 \ + # python-is-python3 \ && rm -rf /var/lib/apt/lists/* @@ -32,6 +35,7 @@ COPY cmake cmake COPY include include COPY test test COPY scripts scripts +COPY pysel pysel COPY CMakeLists.txt sepilinker.cpp ./ # build! From 4245140d60fe700fa81166fb2acc31a981a310fb Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Tue, 16 Jul 2024 07:48:31 +0000 Subject: [PATCH 4/9] :necktie: Add header field for Beamsel --- include/remoteconfiguration.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/remoteconfiguration.cpp b/include/remoteconfiguration.cpp index 3d68f86..9fdfc44 100644 --- a/include/remoteconfiguration.cpp +++ b/include/remoteconfiguration.cpp @@ -93,7 +93,8 @@ void RemoteConfiguration::test_configuration( auto logger{get_logger()}; auto data = client_config.dump(); list headers{"Authorization: "s + m_connection_profile.authenticator.sign_transaction(""), - "Content-Type: application/json" }; + "Content-Type: application/json", + "Beam-Remote: "s + m_remote_id}; string url{assemble_remote_url(this) + "/testConfig/" + client_id}; logger->debug("Sending config test to: {}\n", url); From 208ea570233947878eea9354fbdd4d04ffe2c5ca Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Tue, 16 Jul 2024 08:26:26 +0000 Subject: [PATCH 5/9] :pencil2: Fix typo in Dockerfile --- Dockerfile.ubuntu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 91e90f7..d5b4877 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -1,4 +1,4 @@ -FROM ubuntu:rolling as build +FROM ubuntu:rolling AS build # Backup libs so we know what to copy later to runtime image RUN find /lib/x86_64-linux-gnu/ /usr/lib/x86_64-linux-gnu/ -name '*.so*' > /shared-libs.list From 0146487572245974d8b6e3ef9905e9ca96c444aa Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Wed, 11 Sep 2024 18:53:17 +0000 Subject: [PATCH 6/9] Adapt to Beamsel --- Dockerfile.ubuntu | 4 ++-- include/authenticator.cpp | 6 +++++- include/headerhandlerfunctions.cpp | 8 ++++---- include/jsonhandlerfunctions.cpp | 1 + include/linkagejob.cpp | 3 ++- include/remoteconfiguration.cpp | 7 +++++-- include/secure_epilinker.cpp | 2 +- include/serverhandler.cpp | 1 + 8 files changed, 21 insertions(+), 11 deletions(-) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index d5b4877..2a75547 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -76,8 +76,8 @@ RUN mkdir /log && chown sel:sel /log # find libraries in the correct order ENV LD_LIBRARY_PATH=/lib/x86_64-linux-gnu/:/deps/ -EXPOSE 8161 -EXPOSE 1337-1344 +# EXPOSE 8161 +# EXPOSE 1337-1344 USER sel:sel ENTRYPOINT ["/app/sel"] diff --git a/include/authenticator.cpp b/include/authenticator.cpp index 60e4b39..1c0fff7 100644 --- a/include/authenticator.cpp +++ b/include/authenticator.cpp @@ -75,8 +75,12 @@ SessionResponse Authenticator::check_authentication_header( auto logger = get_logger(); try { string auth_info; - if (auto it = header.find("Authorization"); it != header.end()) { + if (auto it = header.find("Authorization"), it2 = header.find("authorization"); it != header.end() || it2 != header.end()) { + if (it != header.end()) { auth_info = it->second; + } else { + auth_info = it2->second; + } return check_authentication(auth_info); } else { // No auth header auto type{print_auth_type()}; diff --git a/include/headerhandlerfunctions.cpp b/include/headerhandlerfunctions.cpp index bf3ee97..e374f0f 100644 --- a/include/headerhandlerfunctions.cpp +++ b/include/headerhandlerfunctions.cpp @@ -47,16 +47,16 @@ SessionResponse init_mpc(const shared_ptr&, auth_result.return_code != 200){ // auth not ok return auth_result; } - if(header.find("Record-Number") == header.end()) { + if(header.find("record-number") == header.end()) { logger->error("No client record number from {}", remote_id); return responses::status_error(400, "No client record number transmitted"); } - if(header.find("Counting-Mode") == header.end()) { + if(header.find("counting-mode") == header.end()) { counting_mode = false; } aby_server_port = ServerHandler::cget().get_server_port(remote_id); - size_t num_records = stoull(header.find("Record-Number")->second); - counting_mode = header.find("Counting-Mode")->second == "true" ? true : false; + size_t num_records = stoull(header.find("record-number")->second); + counting_mode = header.find("counting-mode")->second == "true" ? true : false; size_t server_record_number; shared_ptr data; try { diff --git a/include/jsonhandlerfunctions.cpp b/include/jsonhandlerfunctions.cpp index b86c93f..12b841a 100644 --- a/include/jsonhandlerfunctions.cpp +++ b/include/jsonhandlerfunctions.cpp @@ -126,6 +126,7 @@ SessionResponse create_job( #ifdef SEL_MATCHING_MODE if(counting_mode){ job->set_counting_job(); + logger->debug("Set job to counting job"); } #endif server_handler.add_linkage_job(remote_id, job); diff --git a/include/linkagejob.cpp b/include/linkagejob.cpp index 1ec805d..800959a 100644 --- a/include/linkagejob.cpp +++ b/include/linkagejob.cpp @@ -177,6 +177,7 @@ size_t LinkageJob::get_server_nvals(size_t num_records) { "Authorization: "s+m_remote_config->get_remote_authenticator().sign_transaction(""), "Record-Number: "s + to_string(num_records), "Counting-Mode: "s + (m_counting_job ? "true" : "false"), + "beam-remote: "s + m_remote_config->get_id(), "Content-Type: application/json"}; string url{assemble_remote_url(m_remote_config) + "/initMPC/"+m_local_config->get_local_id()}; logger->debug("Sending {} request to {}\n",(m_counting_job ? "matching" : "linkage"), url); @@ -186,7 +187,7 @@ size_t LinkageJob::get_server_nvals(size_t num_records) { logger->debug("Response stream:\n{} - {}\n",response.return_code, response.body); // get nvals from response header if (response.return_code == 200) { - return stoull(get_headers(response.body, "Record-Number").front()); + return stoull(get_headers(response.body, "record-number").front()); } else { logger->error("Error communicating with remote epilinker: {} - {}", response.return_code, response.body); } diff --git a/include/remoteconfiguration.cpp b/include/remoteconfiguration.cpp index 9fdfc44..e6e4074 100644 --- a/include/remoteconfiguration.cpp +++ b/include/remoteconfiguration.cpp @@ -94,7 +94,7 @@ void RemoteConfiguration::test_configuration( auto data = client_config.dump(); list headers{"Authorization: "s + m_connection_profile.authenticator.sign_transaction(""), "Content-Type: application/json", - "Beam-Remote: "s + m_remote_id}; + "beam-remote: "s + m_remote_id}; string url{assemble_remote_url(this) + "/testConfig/" + client_id}; logger->debug("Sending config test to: {}\n", url); @@ -109,13 +109,16 @@ void RemoteConfiguration::test_configuration( logger->error("Configuration is not compatible to remote config"); return; } - const auto aby_server_port{get_headers(response.body, "SEL-Port")}; + const auto aby_server_port{get_headers(response.body, "sel-port")}; + logger->debug("Response.body: {}", response.body); + logger->debug("ABY Port: {}", aby_server_port); if (!aby_server_port.empty()) { logger->info("Client registered aby Port {}", aby_server_port.front()); set_aby_port(stoul(aby_server_port.front())); mark_mutually_initialized(); std::thread client_creator([this](){ServerHandler::get().insert_client(m_remote_id);}); client_creator.detach(); + logger->info("Creating client {}", m_remote_id); } } diff --git a/include/secure_epilinker.cpp b/include/secure_epilinker.cpp index 977dbf0..1b72edf 100644 --- a/include/secure_epilinker.cpp +++ b/include/secure_epilinker.cpp @@ -52,7 +52,7 @@ SecureEpilinker::SecureEpilinker(ABYConfig config, CircuitConfig circuit_config) ->GetCircuitBuildRoutine())}, acirc{dynamic_cast(party->GetSharings()[S_ARITH]->GetCircuitBuildRoutine())}, cfg{circuit_config}, selc{make_unique_circuit_builder(cfg, bcirc, ccirc, acirc)} { - get_logger()->debug("SecureEpilinker created."); + get_logger()->debug("SecureEpilinker created:\nRole:{},host:{},port:{}",(int)config.role, config.host, config.port); } // TODO when ABY can separate circuit building/setup/online phases, we create // different SELCircuits per build_circuit()... diff --git a/include/serverhandler.cpp b/include/serverhandler.cpp index df31439..7caef7e 100644 --- a/include/serverhandler.cpp +++ b/include/serverhandler.cpp @@ -112,6 +112,7 @@ void ServerHandler::insert_server(RemoteId id, RemoteAddress remote_address) { void ServerHandler::add_linkage_job(const RemoteId& remote_id, const std::shared_ptr& job){ const auto& config_handler = ConfigurationHandler::cget(); const auto job_id = job->get_id(); + m_logger->debug("Adding job {} with remote {}", job_id, remote_id); if(config_handler.get_remote_config(remote_id)->get_mutual_initialization_status()) { m_client_jobs.emplace(job_id, job); m_worker_threads.at(remote_id).push(job); From 74584a413b9fcd78173826dd62629d6ccd24038f Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Wed, 11 Sep 2024 18:56:38 +0000 Subject: [PATCH 7/9] Dont compile with native instruction set for release builds (for docker images) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c17841..9909d26 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,7 @@ set(${P}_EXTRA_WARNING_FLAGS # Custom compiler and linker flags add_compile_options("-pipe" - "$<$:-O2;-march=native>" + "$<$:-O2>" "$<$:-O0;-g3;-ggdb>" ${CMAKE_FLAGS_EXTRA}) list(APPEND CMAKE_EXE_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS_EXTRA}) From e843cd6b7a7457d7afec50d38fd09d37efbc7d01 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Wed, 11 Sep 2024 18:56:59 +0000 Subject: [PATCH 8/9] Build release version via Docker --- Dockerfile.ubuntu | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 2a75547..38f6549 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -38,6 +38,7 @@ COPY CMakeLists.txt sepilinker.cpp ./ WORKDIR /app/build RUN cmake \ -DCMAKE_FLAGS_EXTRA=-w \ + -DCMAKE_BUILD_TYPE=Release \ .. \ && make sel -j $(nproc) From d53f2238c6e3b7a54f2b947a07fcc1ddb097d171 Mon Sep 17 00:00:00 2001 From: Tobias Kussel Date: Tue, 26 Nov 2024 12:32:47 +0000 Subject: [PATCH 9/9] Fix dockerfile to build correctly --- Dockerfile.ubuntu | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 38f6549..1dc2287 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -1,23 +1,21 @@ -FROM ubuntu:rolling AS build +FROM ubuntu:22.04 AS build # Backup libs so we know what to copy later to runtime image RUN find /lib/x86_64-linux-gnu/ /usr/lib/x86_64-linux-gnu/ -name '*.so*' > /shared-libs.list -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ dpkg-dev \ g++-10 \ - libc6-dev \ libssl-dev \ openssl \ cmake \ - libboost-dev \ - libboost-system-dev \ - libboost-thread-dev \ + libboost-all-dev \ libgmp-dev \ python3 \ libpython3-dev \ libcurl4-openssl-dev \ + python3-distutils \ && rm -rf /var/lib/apt/lists/* # We only installed gcc-10 -> make default @@ -58,7 +56,7 @@ WORKDIR /app/data RUN ../scripts/genkeys.sh . # copy everything into minimal image -FROM ubuntu:rolling +FROM ubuntu:22.04 RUN groupadd -r sel && useradd --no-log-init -r -g sel sel