Rewrite blocking support at runtime & add clang pass

.github/workflows/clang-format.yaml

@@ -8,5 +8,6 @@ jobs:
       - name: Run clang-format style check
         uses: jidicula/clang-format-action@v4.13.0
         with:
-          clang-format-version: '14'
+          clang-format-version: '19'
           check-path: '.'
+          fallback-style: 'Google'

.github/workflows/run-tests.yaml

@@ -9,12 +9,12 @@ jobs:
       run:
         shell: bash
     container: 
-      image: silkeh/clang:18
+      image: silkeh/clang:19
       options: --user root
     timeout-minutes: 10
     steps:
       - name: Install deps
-        run: apt update && apt install -y git ninja-build valgrind libboost-context-dev libgflags-dev
+        run: apt update && apt install -y git ninja-build valgrind libboost-context-dev libgflags-dev libclang-19-dev
       - name: Check out repository code
         uses: actions/checkout@v4
       - name: Build
@@ -29,13 +29,13 @@ jobs:
       run:
         shell: bash
     container: 
-      image: silkeh/clang:18
+      image: silkeh/clang:19
       options: --user root
-    timeout-minutes: 10
+    timeout-minutes: 15
     steps:
       - name: Install deps
         run: |
-          apt update && apt install -y git ninja-build valgrind libgoogle-glog-dev libsnappy-dev protobuf-compiler libboost-context-dev pkg-config libcapstone-dev && \
+          apt update && apt install -y git ninja-build valgrind libgoogle-glog-dev libsnappy-dev protobuf-compiler libboost-context-dev pkg-config libcapstone-dev libclang-19-dev && \
           git clone https://github.com/Kirillog/syscall_intercept.git &&  \
           cmake syscall_intercept -G Ninja -B syscall_intercept/build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang && \
           cmake --build syscall_intercept/build --target install
@@ -46,4 +46,49 @@ jobs:
           cmake -G Ninja -B build -DCMAKE_BUILD_TYPE=RelWithAssert
           cmake --build build --target verify-targets verify-blocking
       - name: "Tests"
-        run: ctest --test-dir build -L "verify" -V
+        run: ctest --parallel 4 --test-dir build -L "verify" -V
+  verifying-folly-release:
+    runs-on: ubuntu-latest
+    env:
+      LTEST_BUILD_PATH: "/__w/LTest/LTest/build"
+    defaults:
+      run:
+        shell: bash
+    container: 
+      image: silkeh/clang:19
+      options: --user root
+    steps:
+      - name: Install deps
+        run: |
+          apt update && apt install -y git ninja-build valgrind libgoogle-glog-dev libsnappy-dev libclang-19-dev \
+                    protobuf-compiler libboost-context-dev pkg-config libcapstone-dev \
+                    libboost-filesystem-dev libboost-program-options-dev libboost-regex-dev \
+                    libdouble-conversion-dev libfast-float-dev libevent-dev libssl-dev libfmt-dev \
+                    libgoogle-glog-dev zlib1g-dev && \
+          git clone --depth=1 https://github.com/Kirillog/syscall_intercept.git &&  \
+          cmake syscall_intercept -G Ninja -B syscall_intercept/build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang && \
+          cmake --build syscall_intercept/build --target install
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Build folly ltest dependencies
+        run: |
+          cmake -G Ninja -B build -DCMAKE_BUILD_TYPE=Release
+          cmake --build build --target plugin_pass runtime
+      - name: Install folly
+        run: |
+          git clone --depth=1 https://github.com/Kirillog/folly.git && \
+          cmake folly -G Ninja -B folly/build_dir -DCMAKE_BUILD_TYPE=Release && \
+          cmake --build folly/build_dir --target install
+      - name: Build tests
+        run: |
+          cmake --build build --target verifying/blocking/folly_rwspinlock verifying/blocking/folly_sharedmutex \
+            verifying/blocking/folly_flatcombining_queue
+      - name: Run folly rwspinlock with pct strategy
+        run: |
+          ./scripts/check.sh 0 ./build/verifying/blocking/folly_rwspinlock --strategy pct --rounds 10000
+      - name: Run folly shared_mutex with pct strategy
+        run: |
+          ./scripts/check.sh 0 ./build/verifying/blocking/folly_sharedmutex --strategy pct --rounds 10000
+      - name: Run folly flatcombining queue with pct strategy
+        run: |
+          ./scripts/check.sh 0 ./build/verifying/blocking/folly_flatcombining_queue --strategy pct --rounds 10000

.gitignore

@@ -7,4 +7,3 @@ venv
 third_party/**
 !third_party/CMakeLists.txt
 dist
-folly

CMakeLists.txt

@@ -7,29 +7,29 @@ set(CMAKE_CXX_STANDARD 20)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-# TODO(kmitkin): require to understand, what is it considered to be "optimized" build
-# set(CMAKE_CXX_FLAGS_RELEASE "???")
-set(CMAKE_CXX_FLAGS_DEBUG "-g -ggdb3 -O0 -fno-omit-frame-pointer")
-# set(CMAKE_CXX_FLAGS "-stdlib=libc++ ${CMAKE_CXX_FLAGS}")
+set(APPLY_CLANG_TOOL ON)
 
 set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithAssert" CACHE STRING "" FORCE)
 
-
 set(CMAKE_C_FLAGS_RELWITHASSERT "${CMAKE_C_FLAGS_RELEASE} -UNDEBUG" CACHE STRING "" FORCE)
 set(CMAKE_CXX_FLAGS_RELWITHASSERT "${CMAKE_CXX_FLAGS_RELEASE} -UNDEBUG" CACHE STRING "" FORCE)
 
+
+set(CMAKE_CXX_FLAGS_DEBUG "-DDEBUG -g -ggdb3 -O0 -fno-omit-frame-pointer")
 set(CMAKE_WARN_FLAGS -Wall -Wextra -Werror -pedantic-errors)
+
 if(CMAKE_BUILD_TYPE MATCHES Debug)
     message(STATUS "Debug mode ON")
     set(CMAKE_ASAN_FLAGS -fsanitize=address -fsanitize=undefined -DADDRESS_SANITIZER)
 endif(CMAKE_BUILD_TYPE MATCHES Debug)
 
 add_subdirectory(third_party)
-
+include(CTest)
 include(GoogleTest)
 fuzztest_setup_fuzzing_flags()
 enable_testing()
 
+add_subdirectory(clangpass)
 add_subdirectory(codegen)
 add_subdirectory(runtime)
 

Dockerfile

@@ -1,13 +1,26 @@
 FROM silkeh/clang:19 AS ltest
 
-RUN apt update && apt install -y git ninja-build valgrind libboost-context-dev libgflags-dev libstdc++-11-dev
+RUN apt update && apt install -y git ninja-build valgrind libgflags-dev libstdc++-11-dev libclang-19-dev
 RUN mv /usr/lib/gcc/x86_64-linux-gnu/12 /usr/lib/gcc/x86_64-linux-gnu/_12
 
 FROM ltest as blocking
-RUN apt install -y pkg-config libcapstone-dev && \
+RUN apt install -y pkg-config libcapstone-dev libboost-context-dev && \
     git clone https://github.com/Kirillog/syscall_intercept.git &&  \
     cmake syscall_intercept -G Ninja -B syscall_intercept/build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang && \
     cmake --build syscall_intercept/build --target install
+FROM blocking as folly-blocking
+RUN apt install -y libboost-filesystem-dev libboost-program-options-dev libboost-regex-dev \
+                    libdouble-conversion-dev libfast-float-dev libevent-dev libssl-dev libfmt-dev \
+                    libgoogle-glog-dev zlib1g-dev && \
+    git clone https://github.com/Kirillog/folly.git && \
+    cmake folly -G Ninja -B folly/build_dir -DCMAKE_BUILD_TYPE=Release
+    # cmake --build folly/build_dir --target install
+FROM ltest as userver-blocking
+# userver conflicts with default libboost-context-dev (1.74) version, 1.81 required
+RUN apt install -y python3-dev python3-venv \
+        libboost-context1.81-dev libboost-filesystem1.81-dev libboost-program-options1.81-dev libboost-regex1.81-dev 
+        libboost-stacktrace1.81-dev libboost-locale1.81-dev \
+        libzstd-dev libyaml-cpp-dev libcrypto++-dev libnghttp2-dev libev-dev
 RUN sh -c "$(wget -O- https://github.com/deluan/zsh-in-docker/releases/download/v1.2.1/zsh-in-docker.sh)" -- \
        -p git
 CMD [ "zsh" ]

clangpass/CMakeLists.txt

@@ -0,0 +1,38 @@
+#===============================================================================
+# SETUP CLANG PLUGIN
+#===============================================================================
+find_package(Clang REQUIRED CONFIG)
+if("${LLVM_VERSION_MAJOR}" VERSION_LESS 19)
+  message(FATAL_ERROR "Found LLVM ${LLVM_VERSION_MAJOR}, but need LLVM 19 or above")
+endif()
+
+include_directories(SYSTEM "${LLVM_INCLUDE_DIRS};${CLANG_INCLUDE_DIRS}")
+#===============================================================================
+# SETUP CLANG TOOL
+#===============================================================================
+set(CLANG_TOOL "ClangPassTool")
+set(CLANG_TOOL_SOURCES
+  "${CMAKE_CURRENT_SOURCE_DIR}/clangpass_tool.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/ast_consumer.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/refactor_matcher.cpp"
+)
+
+add_executable(
+  ${CLANG_TOOL}
+  ${CLANG_TOOL_SOURCES}
+)
+
+# Configure include directories for 'tool'
+target_include_directories(
+  ${CLANG_TOOL}
+  PRIVATE
+  "${CMAKE_CURRENT_SOURCE_DIR}/include"
+)
+
+# Link in the required libraries 
+target_link_libraries(
+  ${CLANG_TOOL}
+  PRIVATE
+  clangTooling
+  clangToolingRefactoring
+)

clangpass/ast_consumer.cpp

@@ -0,0 +1,71 @@
+
+#include "include/clangpass.h"
+
+replace_pass::CodeRefactorASTConsumer::CodeRefactorASTConsumer(
+    ASTContext& context, clang::Rewriter& rewriter,
+    std::vector<ReplacePair> names, std::string temp_prefix)
+    : refactor_handler(context, rewriter, names),
+      names(names),
+      rewriter(rewriter),
+      temp_prefix(std::move(temp_prefix)) {
+  auto factory = NameMatcherFactory();
+
+  for (auto name : names) {
+    const auto elaborated_matcher = factory.CreateMatcherFor(name.old_name);
+    // Uses previous matcher inside, but returns a wrapping QualifiedTypeLoc
+    // node which is used in the function parameters
+    const auto qualified_matcher =
+        qualifiedTypeLoc(hasUnqualifiedLoc(elaborated_matcher));
+    match_finder.addMatcher(
+        elaborated_matcher.bind("ElaboratedTypeLoc" + name.old_name),
+        &refactor_handler);
+    match_finder.addMatcher(
+        qualified_matcher.bind("QualifiedTypeLoc" + name.old_name),
+        &refactor_handler);
+  }
+}
+
+std::string replace_pass::CodeRefactorASTConsumer::RefactoredFileName(
+    StringRef original_filename) const {
+  size_t slash_index = original_filename.rfind("/");
+  // the path should be absolute, so in the worst case we will get '/' as index
+  // 0
+  assert(slash_index != std::string::npos);
+  slash_index += 1;  // include the '/' itself
+
+  std::string path = std::string(original_filename.begin(),
+                                 original_filename.begin() + slash_index);
+  std::string filename = std::string(original_filename.begin() + slash_index,
+                                     original_filename.end());
+
+  return path + temp_prefix + filename;
+}
+
+void replace_pass::CodeRefactorASTConsumer::HandleTranslationUnit(
+    clang::ASTContext& ctx) {
+  match_finder.matchAST(ctx);
+  const SourceManager& source_manager = rewriter.getSourceMgr();
+  auto fileID = source_manager.getMainFileID();
+  const RewriteBuffer& buffer = rewriter.getEditBuffer(fileID);
+
+  // Output to stdout
+  llvm::outs() << "Transformed code:\n";
+  buffer.write(llvm::outs());
+  llvm::outs() << "\n";
+
+  // Output to file
+  const FileEntry* entry = source_manager.getFileEntryForID(fileID);
+  std::string new_filename = RefactoredFileName(entry->tryGetRealPathName());
+
+  std::error_code code;
+  llvm::raw_fd_ostream ostream(new_filename, code, llvm::sys::fs::OF_None);
+  if (code) {
+    llvm::errs() << "Error: Could not open output file: " << code.message()
+                 << "\n";
+    return;
+  }
+
+  llvm::outs() << "Writing to file: " << new_filename << "\n";
+  buffer.write(ostream);
+  ostream.close();
+}

clangpass/clangpass_tool.cpp

@@ -0,0 +1,107 @@
+//==============================================================================
+// FILE:
+//    clangpass_tool.cpp
+//
+// DESCRIPTION:
+//    Replaces all ::std:: names usages to custom runtime friendly
+//    implementations
+//
+// USAGE:
+//    * ./build/bin/clangpass_tool --replace-names=::std::atomic,::std::mutex
+//    --insert-names=LTestAtomic,ltest::mutex
+//    ./verifying/targets/nonlinear_queue.cpp
+//
+// License: The Unlicense
+//==============================================================================
+#include <clang/Basic/Diagnostic.h>
+#include <clang/Rewrite/Core/Rewriter.h>
+#include <clang/Tooling/Refactoring/Rename/RenamingAction.h>
+#include <clang/Tooling/Refactoring/Rename/USRFindingAction.h>
+
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Refactoring.h"
+#include "include/clangpass.h"
+#include "llvm/Support/CommandLine.h"
+
+using namespace clang;
+using namespace replace_pass;
+
+//===----------------------------------------------------------------------===//
+// Command line options
+//===----------------------------------------------------------------------===//
+static cl::OptionCategory CodeRefactorCategory("atomics-replacer options");
+
+static cl::opt<std::string> TemporaryPrefix{
+    "temp-prefix", cl::desc("Prefix for temporary files"), cl::init("__tmp_"),
+    cl::cat(CodeRefactorCategory)};
+static cl::list<std::string> ClassNamesToReplace{
+    "replace-names",
+    cl::desc("Names of the classes/structs which usages should be renamed"),
+    cl::OneOrMore, cl::CommaSeparated, cl::cat(CodeRefactorCategory)};
+static cl::list<std::string> ClassNamesToInsert{
+    "insert-names",
+    cl::desc("Names of the classes/structs which should be used instead"),
+    cl::OneOrMore, cl::CommaSeparated, cl::cat(CodeRefactorCategory)};
+
+//===----------------------------------------------------------------------===//
+// PluginASTAction
+//===----------------------------------------------------------------------===//
+class CodeRefactorPluginAction : public PluginASTAction {
+ public:
+  explicit CodeRefactorPluginAction() {}
+  // Not used
+  bool ParseArgs(const CompilerInstance &compiler,
+                 const std::vector<std::string> &args) override {
+    return true;
+  }
+
+  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &compiler,
+                                                 StringRef file) override {
+    rewriter.setSourceMgr(compiler.getSourceManager(), compiler.getLangOpts());
+
+    std::vector<ReplacePair> pairs;
+    pairs.reserve(ClassNamesToReplace.size());
+    for (int i = 0; i < ClassNamesToReplace.size(); ++i) {
+      pairs.emplace_back(ClassNamesToReplace[i], ClassNamesToInsert[i]);
+    }
+
+    return std::make_unique<CodeRefactorASTConsumer>(
+        compiler.getASTContext(), rewriter, pairs, TemporaryPrefix);
+  }
+
+ private:
+  Rewriter rewriter;
+};
+
+//===----------------------------------------------------------------------===//
+// Main driver code.
+//===----------------------------------------------------------------------===//
+int main(int argc, const char **argv) {
+  Expected<tooling::CommonOptionsParser> options =
+      clang::tooling::CommonOptionsParser::create(argc, argv,
+                                                  CodeRefactorCategory);
+  if (auto E = options.takeError()) {
+    errs() << "Problem constructing CommonOptionsParser "
+           << toString(std::move(E)) << '\n';
+    return EXIT_FAILURE;
+  }
+
+  // auto files = eOptParser->getSourcePathList();
+  // std::vector<const char*> Args = {"clang++", "-E" };
+  // Args.insert(Args.end(), files.begin(), files.end());
+
+  // auto OptionsParser = clang::tooling::CommonOptionsParser::create(
+  //     Args.size(), Args.data(),
+  //   );
+
+  // clang::tooling::ClangTool Tool(OptionsParser->getCompilations(),
+  //                              OptionsParser->getSourcePathList());
+
+  clang::tooling::RefactoringTool tool(options->getCompilations(),
+                                       options->getSourcePathList());
+
+  return tool.run(
+      clang::tooling::newFrontendActionFactory<CodeRefactorPluginAction>()
+          .get());
+}