Skip to content

Commit 992c695

Browse files
committed
add folly_sharedmutex
extend switch point insertion fix hook & folly shared mutex try to setup CI to run folly tests refactor folly_rwspinlock add bank && custom blocking primitives format code speed up futex blocking && change release schema forbid CoroYield calls in inappropriate contexts fix CI add flatcombining queue use renaming action clang_pass: support simple names & several match types refactor enable CI fix livelock problem && get rid of legacy token API format code fix pct backoff strategy add ad-hoc termination order refactor verifying targets fix after rebasing onto minimization patch refactor clang pass rewrite totally blocking support at runtime fix livelock avoiding reformat code && enable testing fix formatting scripts && reformat code fix review comments add conditional variables testing add conditional variable primitive fix futex_queues.PopAll pct_strategy: run round robin for livelock avoding add deadlock detection refactor code refactor StrategyVerifier extend buffered channel test
1 parent d415754 commit 992c695

File tree

83 files changed

+2096
-1020
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

83 files changed

+2096
-1020
lines changed

.github/workflows/clang-format.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,6 @@ jobs:
88
- name: Run clang-format style check
99
uses: jidicula/clang-format-action@v4.13.0
1010
with:
11-
clang-format-version: '14'
11+
clang-format-version: '19'
1212
check-path: '.'
13+
fallback-style: 'Google'

.github/workflows/run-tests.yaml

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,12 @@ jobs:
99
run:
1010
shell: bash
1111
container:
12-
image: silkeh/clang:18
12+
image: silkeh/clang:19
1313
options: --user root
1414
timeout-minutes: 10
1515
steps:
1616
- name: Install deps
17-
run: apt update && apt install -y git ninja-build valgrind libboost-context-dev libgflags-dev
17+
run: apt update && apt install -y git ninja-build valgrind libboost-context-dev libgflags-dev libclang-19-dev
1818
- name: Check out repository code
1919
uses: actions/checkout@v4
2020
- name: Build
@@ -29,13 +29,13 @@ jobs:
2929
run:
3030
shell: bash
3131
container:
32-
image: silkeh/clang:18
32+
image: silkeh/clang:19
3333
options: --user root
3434
timeout-minutes: 10
3535
steps:
3636
- name: Install deps
3737
run: |
38-
apt update && apt install -y git ninja-build valgrind libgoogle-glog-dev libsnappy-dev protobuf-compiler libboost-context-dev pkg-config libcapstone-dev && \
38+
apt update && apt install -y git ninja-build valgrind libgoogle-glog-dev libsnappy-dev protobuf-compiler libboost-context-dev pkg-config libcapstone-dev libclang-19-dev && \
3939
git clone https://github.com/Kirillog/syscall_intercept.git && \
4040
cmake syscall_intercept -G Ninja -B syscall_intercept/build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang && \
4141
cmake --build syscall_intercept/build --target install
@@ -46,4 +46,49 @@ jobs:
4646
cmake -G Ninja -B build -DCMAKE_BUILD_TYPE=RelWithAssert
4747
cmake --build build --target verify-targets verify-blocking
4848
- name: "Tests"
49-
run: ctest --test-dir build -L "verify" -V
49+
run: ctest --parallel 4 --test-dir build -L "verify" -V
50+
verifying-folly-release:
51+
runs-on: ubuntu-latest
52+
env:
53+
LTEST_BUILD_PATH: "/__w/LTest/LTest/build"
54+
defaults:
55+
run:
56+
shell: bash
57+
container:
58+
image: silkeh/clang:19
59+
options: --user root
60+
steps:
61+
- name: Install deps
62+
run: |
63+
apt update && apt install -y git ninja-build valgrind libgoogle-glog-dev libsnappy-dev libclang-19-dev \
64+
protobuf-compiler libboost-context-dev pkg-config libcapstone-dev \
65+
libboost-filesystem-dev libboost-program-options-dev libboost-regex-dev \
66+
libdouble-conversion-dev libfast-float-dev libevent-dev libssl-dev libfmt-dev \
67+
libgoogle-glog-dev zlib1g-dev && \
68+
git clone --depth=1 https://github.com/Kirillog/syscall_intercept.git && \
69+
cmake syscall_intercept -G Ninja -B syscall_intercept/build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang && \
70+
cmake --build syscall_intercept/build --target install
71+
- name: Check out repository code
72+
uses: actions/checkout@v4
73+
- name: Build folly ltest dependencies
74+
run: |
75+
cmake -G Ninja -B build -DCMAKE_BUILD_TYPE=Release
76+
cmake --build build --target plugin_pass runtime
77+
- name: Install folly
78+
run: |
79+
git clone --depth=1 https://github.com/Kirillog/folly.git && \
80+
cmake folly -G Ninja -B folly/build_dir -DCMAKE_BUILD_TYPE=Release && \
81+
cmake --build folly/build_dir --target install
82+
- name: Build tests
83+
run: |
84+
cmake --build build --target verifying/blocking/folly_rwspinlock verifying/blocking/folly_sharedmutex \
85+
verifying/blocking/folly_flatcombining_queue
86+
- name: Run folly rwspinlock with pct strategy
87+
run: |
88+
./scripts/check.sh 0 ./build/verifying/blocking/folly_rwspinlock --strategy pct --rounds 10000
89+
- name: Run folly shared_mutex with pct strategy
90+
run: |
91+
./scripts/check.sh 0 ./build/verifying/blocking/folly_sharedmutex --strategy pct --rounds 10000
92+
- name: Run folly flatcombining queue with pct strategy
93+
run: |
94+
./scripts/check.sh 0 ./build/verifying/blocking/folly_flatcombining_queue --strategy pct --rounds 10000

.gitignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,3 @@ venv
77
third_party/**
88
!third_party/CMakeLists.txt
99
dist
10-
folly

CMakeLists.txt

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,29 +7,34 @@ set(CMAKE_CXX_STANDARD 20)
77

88
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
99

10+
set(APPLY_CLANG_TOOL ON)
11+
1012
# TODO(kmitkin): require to understand, what is it considered to be "optimized" build
1113
# set(CMAKE_CXX_FLAGS_RELEASE "???")
12-
set(CMAKE_CXX_FLAGS_DEBUG "-g -ggdb3 -O0 -fno-omit-frame-pointer")
13-
# set(CMAKE_CXX_FLAGS "-stdlib=libc++ ${CMAKE_CXX_FLAGS}")
14+
# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
1415

1516
set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithAssert" CACHE STRING "" FORCE)
1617

1718

1819
set(CMAKE_C_FLAGS_RELWITHASSERT "${CMAKE_C_FLAGS_RELEASE} -UNDEBUG" CACHE STRING "" FORCE)
1920
set(CMAKE_CXX_FLAGS_RELWITHASSERT "${CMAKE_CXX_FLAGS_RELEASE} -UNDEBUG" CACHE STRING "" FORCE)
2021

22+
23+
set(CMAKE_CXX_FLAGS_DEBUG "-DDEBUG -g -ggdb3 -O0 -fno-omit-frame-pointer")
2124
set(CMAKE_WARN_FLAGS -Wall -Wextra -Werror -pedantic-errors)
25+
2226
if(CMAKE_BUILD_TYPE MATCHES Debug)
2327
message(STATUS "Debug mode ON")
2428
set(CMAKE_ASAN_FLAGS -fsanitize=address -fsanitize=undefined -DADDRESS_SANITIZER)
2529
endif(CMAKE_BUILD_TYPE MATCHES Debug)
2630

2731
add_subdirectory(third_party)
28-
32+
include(CTest)
2933
include(GoogleTest)
3034
fuzztest_setup_fuzzing_flags()
3135
enable_testing()
3236

37+
add_subdirectory(clangpass)
3338
add_subdirectory(codegen)
3439
add_subdirectory(runtime)
3540

Dockerfile

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,26 @@
11
FROM silkeh/clang:19 AS ltest
22

3-
RUN apt update && apt install -y git ninja-build valgrind libboost-context-dev libgflags-dev libstdc++-11-dev
3+
RUN apt update && apt install -y git ninja-build valgrind libgflags-dev libstdc++-11-dev libclang-19-dev
44
RUN mv /usr/lib/gcc/x86_64-linux-gnu/12 /usr/lib/gcc/x86_64-linux-gnu/_12
55

66
FROM ltest as blocking
7-
RUN apt install -y pkg-config libcapstone-dev && \
7+
RUN apt install -y pkg-config libcapstone-dev libboost-context-dev && \
88
git clone https://github.com/Kirillog/syscall_intercept.git && \
99
cmake syscall_intercept -G Ninja -B syscall_intercept/build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang && \
1010
cmake --build syscall_intercept/build --target install
11+
FROM blocking as folly-blocking
12+
RUN apt install -y libboost-filesystem-dev libboost-program-options-dev libboost-regex-dev \
13+
libdouble-conversion-dev libfast-float-dev libevent-dev libssl-dev libfmt-dev \
14+
libgoogle-glog-dev zlib1g-dev && \
15+
git clone https://github.com/Kirillog/folly.git && \
16+
cmake folly -G Ninja -B folly/build_dir -DCMAKE_BUILD_TYPE=Release
17+
# cmake --build folly/build_dir --target install
18+
FROM ltest as userver-blocking
19+
# userver conflicts with default libboost-context-dev (1.74) version, 1.81 required
20+
RUN apt install -y python3-dev python3-venv \
21+
libboost-context1.81-dev libboost-filesystem1.81-dev libboost-program-options1.81-dev libboost-regex1.81-dev
22+
libboost-stacktrace1.81-dev libboost-locale1.81-dev \
23+
libzstd-dev libyaml-cpp-dev libcrypto++-dev libnghttp2-dev libev-dev
1124
RUN sh -c "$(wget -O- https://github.com/deluan/zsh-in-docker/releases/download/v1.2.1/zsh-in-docker.sh)" -- \
1225
-p git
1326
CMD [ "zsh" ]

clangpass/CMakeLists.txt

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#===============================================================================
2+
# SETUP CLANG PLUGIN
3+
#===============================================================================
4+
find_package(Clang REQUIRED CONFIG)
5+
if("${LLVM_VERSION_MAJOR}" VERSION_LESS 19)
6+
message(FATAL_ERROR "Found LLVM ${LLVM_VERSION_MAJOR}, but need LLVM 19 or above")
7+
endif()
8+
9+
include_directories(SYSTEM "${LLVM_INCLUDE_DIRS};${CLANG_INCLUDE_DIRS}")
10+
#===============================================================================
11+
# SETUP CLANG TOOL
12+
#===============================================================================
13+
set(CLANG_TOOL "ClangPassTool")
14+
set(CLANG_TOOL_SOURCES
15+
"${CMAKE_CURRENT_SOURCE_DIR}/clangpass_tool.cpp"
16+
"${CMAKE_CURRENT_SOURCE_DIR}/ast_consumer.cpp"
17+
"${CMAKE_CURRENT_SOURCE_DIR}/refactor_matcher.cpp"
18+
)
19+
20+
add_executable(
21+
${CLANG_TOOL}
22+
${CLANG_TOOL_SOURCES}
23+
)
24+
25+
# Configure include directories for 'tool'
26+
target_include_directories(
27+
${CLANG_TOOL}
28+
PRIVATE
29+
"${CMAKE_CURRENT_SOURCE_DIR}/include"
30+
)
31+
32+
# Link in the required libraries
33+
target_link_libraries(
34+
${CLANG_TOOL}
35+
PRIVATE
36+
clangTooling
37+
clangToolingRefactoring
38+
)

clangpass/ast_consumer.cpp

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
2+
#include "include/clangpass.h"
3+
4+
replace_pass::CodeRefactorASTConsumer::CodeRefactorASTConsumer(
5+
ASTContext& context, clang::Rewriter& rewriter,
6+
std::vector<ReplacePair> names, std::string temp_prefix)
7+
: refactor_handler(context, rewriter, names),
8+
names(names),
9+
rewriter(rewriter),
10+
temp_prefix(std::move(temp_prefix)) {
11+
auto factory = NameMatcherFactory();
12+
13+
for (auto name : names) {
14+
const auto elaborated_matcher = factory.CreateMatcherFor(name.old_name);
15+
// Uses previous matcher inside, but returns a wrapping QualifiedTypeLoc
16+
// node which is used in the function parameters
17+
const auto qualified_matcher =
18+
qualifiedTypeLoc(hasUnqualifiedLoc(elaborated_matcher));
19+
match_finder.addMatcher(
20+
elaborated_matcher.bind("ElaboratedTypeLoc" + name.old_name),
21+
&refactor_handler);
22+
match_finder.addMatcher(
23+
qualified_matcher.bind("QualifiedTypeLoc" + name.old_name),
24+
&refactor_handler);
25+
}
26+
}
27+
28+
std::string replace_pass::CodeRefactorASTConsumer::RefactoredFileName(
29+
StringRef original_filename) const {
30+
size_t slash_index = original_filename.rfind("/");
31+
// the path should be absolute, so in the worst case we will get '/' as index
32+
// 0
33+
assert(slash_index != std::string::npos);
34+
slash_index += 1; // include the '/' itself
35+
36+
std::string path = std::string(original_filename.begin(),
37+
original_filename.begin() + slash_index);
38+
std::string filename = std::string(original_filename.begin() + slash_index,
39+
original_filename.end());
40+
41+
return path + temp_prefix + filename;
42+
}
43+
44+
void replace_pass::CodeRefactorASTConsumer::HandleTranslationUnit(
45+
clang::ASTContext& ctx) {
46+
match_finder.matchAST(ctx);
47+
const SourceManager& source_manager = rewriter.getSourceMgr();
48+
auto fileID = source_manager.getMainFileID();
49+
const RewriteBuffer& buffer = rewriter.getEditBuffer(fileID);
50+
51+
// Output to stdout
52+
llvm::outs() << "Transformed code:\n";
53+
buffer.write(llvm::outs());
54+
llvm::outs() << "\n";
55+
56+
// Output to file
57+
const FileEntry* entry = source_manager.getFileEntryForID(fileID);
58+
std::string new_filename = RefactoredFileName(entry->tryGetRealPathName());
59+
60+
std::error_code code;
61+
llvm::raw_fd_ostream ostream(new_filename, code, llvm::sys::fs::OF_None);
62+
if (code) {
63+
llvm::errs() << "Error: Could not open output file: " << code.message()
64+
<< "\n";
65+
return;
66+
}
67+
68+
llvm::outs() << "Writing to file: " << new_filename << "\n";
69+
buffer.write(ostream);
70+
ostream.close();
71+
}

clangpass/clangpass_tool.cpp

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
//==============================================================================
2+
// FILE:
3+
// clangpass_tool.cpp
4+
//
5+
// DESCRIPTION:
6+
// Replaces all ::std:: names usages to custom runtime friendly
7+
// implementations
8+
//
9+
// USAGE:
10+
// * ./build/bin/clangpass_tool --replace-names=::std::atomic,::std::mutex
11+
// --insert-names=LTestAtomic,ltest::mutex
12+
// ./verifying/targets/nonlinear_queue.cpp
13+
//
14+
// License: The Unlicense
15+
//==============================================================================
16+
#include <clang/Basic/Diagnostic.h>
17+
#include <clang/Rewrite/Core/Rewriter.h>
18+
#include <clang/Tooling/Refactoring/Rename/RenamingAction.h>
19+
#include <clang/Tooling/Refactoring/Rename/USRFindingAction.h>
20+
21+
#include "clang/Frontend/CompilerInstance.h"
22+
#include "clang/Tooling/CommonOptionsParser.h"
23+
#include "clang/Tooling/Refactoring.h"
24+
#include "include/clangpass.h"
25+
#include "llvm/Support/CommandLine.h"
26+
27+
using namespace clang;
28+
using namespace replace_pass;
29+
30+
//===----------------------------------------------------------------------===//
31+
// Command line options
32+
//===----------------------------------------------------------------------===//
33+
static cl::OptionCategory CodeRefactorCategory("atomics-replacer options");
34+
35+
static cl::opt<std::string> TemporaryPrefix{
36+
"temp-prefix", cl::desc("Prefix for temporary files"), cl::init("__tmp_"),
37+
cl::cat(CodeRefactorCategory)};
38+
static cl::list<std::string> ClassNamesToReplace{
39+
"replace-names",
40+
cl::desc("Names of the classes/structs which usages should be renamed"),
41+
cl::OneOrMore, cl::CommaSeparated, cl::cat(CodeRefactorCategory)};
42+
static cl::list<std::string> ClassNamesToInsert{
43+
"insert-names",
44+
cl::desc("Names of the classes/structs which should be used instead"),
45+
cl::OneOrMore, cl::CommaSeparated, cl::cat(CodeRefactorCategory)};
46+
47+
//===----------------------------------------------------------------------===//
48+
// PluginASTAction
49+
//===----------------------------------------------------------------------===//
50+
class CodeRefactorPluginAction : public PluginASTAction {
51+
public:
52+
explicit CodeRefactorPluginAction() {}
53+
// Not used
54+
bool ParseArgs(const CompilerInstance &compiler,
55+
const std::vector<std::string> &args) override {
56+
return true;
57+
}
58+
59+
std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &compiler,
60+
StringRef file) override {
61+
rewriter.setSourceMgr(compiler.getSourceManager(), compiler.getLangOpts());
62+
63+
std::vector<ReplacePair> pairs;
64+
pairs.reserve(ClassNamesToReplace.size());
65+
for (int i = 0; i < ClassNamesToReplace.size(); ++i) {
66+
pairs.emplace_back(ClassNamesToReplace[i], ClassNamesToInsert[i]);
67+
}
68+
69+
return std::make_unique<CodeRefactorASTConsumer>(
70+
compiler.getASTContext(), rewriter, pairs, TemporaryPrefix);
71+
}
72+
73+
private:
74+
Rewriter rewriter;
75+
};
76+
77+
//===----------------------------------------------------------------------===//
78+
// Main driver code.
79+
//===----------------------------------------------------------------------===//
80+
int main(int argc, const char **argv) {
81+
Expected<tooling::CommonOptionsParser> options =
82+
clang::tooling::CommonOptionsParser::create(argc, argv,
83+
CodeRefactorCategory);
84+
if (auto E = options.takeError()) {
85+
errs() << "Problem constructing CommonOptionsParser "
86+
<< toString(std::move(E)) << '\n';
87+
return EXIT_FAILURE;
88+
}
89+
90+
// auto files = eOptParser->getSourcePathList();
91+
// std::vector<const char*> Args = {"clang++", "-E" };
92+
// Args.insert(Args.end(), files.begin(), files.end());
93+
94+
// auto OptionsParser = clang::tooling::CommonOptionsParser::create(
95+
// Args.size(), Args.data(),
96+
// );
97+
98+
// clang::tooling::ClangTool Tool(OptionsParser->getCompilations(),
99+
// OptionsParser->getSourcePathList());
100+
101+
clang::tooling::RefactoringTool tool(options->getCompilations(),
102+
options->getSourcePathList());
103+
104+
return tool.run(
105+
clang::tooling::newFrontendActionFactory<CodeRefactorPluginAction>()
106+
.get());
107+
}

0 commit comments

Comments
 (0)