diff --git a/ci/cli-test.sh b/ci/cli-test.sh new file mode 100755 index 0000000..79605e9 --- /dev/null +++ b/ci/cli-test.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -e +set -x + +CLI_CMD=zlog +LMDB_DIR=$(mktemp -d) + +INPUT_FILE=$(mktemp) +OUTPUT_FILE=$(mktemp) +EXPECTED_FILE=$(mktemp) +trap "rm -rf ${LMDB_DIR} ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECTED_FILE}" exit + +${CLI_CMD} --backend lmdb --db-path ${LMDB_DIR} log create testlog +! ${CLI_CMD} --backend lmdb --db-path ${LMDB_DIR} log create testlog + +${CLI_CMD} --backend lmdb --db-path ${LMDB_DIR} log append testlog 'just a lil test' +${CLI_CMD} --backend lmdb --db-path ${LMDB_DIR} log append testlog 'and another' +${CLI_CMD} --backend lmdb --db-path ${LMDB_DIR} log append testlog 'one more and were done' + +echo '616e6420616e6f74686572' >> ${EXPECTED_FILE} +${CLI_CMD} --backend lmdb --db-path ${LMDB_DIR} log read testlog 1 >> ${OUTPUT_FILE} +diff ${EXPECTED_FILE} ${OUTPUT_FILE} + +> ${EXPECTED_FILE} +> ${OUTPUT_FILE} + +echo 'arthur willey!' >> ${INPUT_FILE} +echo 'nautili hunter' >> ${INPUT_FILE} +echo 'abcdefghijlmno' >> ${INPUT_FILE} + +${CLI_CMD} --backend lmdb --db-path ${LMDB_DIR} log append testlog -i ${INPUT_FILE} + +echo '0: 6a7573742061206c696c2074657374' >> ${EXPECTED_FILE} +echo '1: 616e6420616e6f74686572' >> ${EXPECTED_FILE} +echo '2: 6f6e65206d6f726520616e64207765726520646f6e65' >> ${EXPECTED_FILE} +echo '3: 6172746875722077696c6c6579210a6e617574696c692068756e7465720a6162636465666768696a6c6d6e6f0a' >> ${EXPECTED_FILE} + +${CLI_CMD} --backend lmdb --db-path ${LMDB_DIR} log dump testlog >> ${OUTPUT_FILE} + +diff ${EXPECTED_FILE} ${OUTPUT_FILE} + +${CLI_CMD} --backend lmdb --db-path ${LMDB_DIR} log fill testlog 30 +! ${CLI_CMD} --backend lmdb --db-path ${LMDB_DIR} log fill testlog 1 + +${CLI_CMD} --backend lmdb --db-path ${LMDB_DIR} log trim testlog 2 + diff --git a/ci/test.sh b/ci/test.sh index 15c263e..9fddabb 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -100,3 +100,8 @@ if [[ "$OSTYPE" != "darwin"* && "$JNI" == "ON" ]]; then popd fi + +# test cli +export PATH +${ZLOG_DIR}/ci/cli-test.sh + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 64bfe49..cd513a2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -63,3 +63,4 @@ target_link_libraries(zlog ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_SYSTEM_LIBRARY} ) +install(TARGETS zlog DESTINATION bin) diff --git a/src/zlog.cc b/src/zlog.cc index 24e89ee..c66d40e 100644 --- a/src/zlog.cc +++ b/src/zlog.cc @@ -1,14 +1,19 @@ +#include #include #include +#include #include #include #include "zlog/backend.h" +#include "zlog/log.h" #include "zlog/options.h" #include "libzlog/striper.h" #include "proto/zlog.pb.h" namespace po = boost::program_options; +int handle_log(std::vector, std::shared_ptr, std::string); + int main(int argc, char **argv) { std::vector command; @@ -16,6 +21,7 @@ int main(int argc, char **argv) std::string backend_name; std::string pool; std::string db_path; + std::string input_filename; po::options_description opts("Benchmark options"); opts.add_options() @@ -25,21 +31,27 @@ int main(int argc, char **argv) ("pool", po::value(&pool)->default_value("zlog"), "pool (ceph)") ("db-path", po::value(&db_path)->default_value("/tmp/zlog.bench.db"), "db path (lmdb)") ("command", po::value>(&command), "command") + ("input-file,i", po::value(&input_filename), "input filename for log append") ; + // This gives us a vector of the command line arguments with flags removed po::positional_options_description popts; popts.add("command", -1); po::variables_map vm; - po::store(po::command_line_parser(argc, argv).options(opts).positional(popts).run(), vm); + try { + po::store(po::command_line_parser(argc, argv).options(opts).positional(popts).run(), vm); + po::notify(vm); + } catch (const boost::program_options::error &exception) { + std::cerr << exception.what() << std::endl; + return 1; + } if (vm.count("help")) { std::cout << opts << std::endl; return 1; } - po::notify(vm); - zlog::Options options; options.backend_name = backend_name; @@ -59,47 +71,263 @@ int main(int argc, char **argv) return ret; } - std::string hoid; - std::string prefix; - ret = backend->OpenLog(log_name, &hoid, &prefix); - if (ret) { - std::cerr << "backend::openlog " << ret << std::endl; - return ret; + if (command.size() > 0) { + if (command[0] == "log") { + auto subcommand = std::vector(command.begin() + 1, command.end()); + return handle_log(subcommand, backend, input_filename); + } } - uint64_t epoch = 1; - while (true) { - std::map views; - ret = backend->ReadViews(hoid, epoch, 1, &views); - if (ret) { - std::cerr << "read views error " << ret << std::endl; + if (command == std::vector{ "link", "list" }) { + std::vector links; + int ret = backend->ListLinks(links); + if (ret != 0) { + std::cerr << "backend::ListLinks " << ret << std::endl; return ret; } - - if (views.empty()) { - break; + for (const auto &link : links) { + std::cout << link << std::endl; + } + return 0; + } else if (command == std::vector{ "head", "list" }) { + std::vector heads; + int ret = backend->ListHeads(heads); + if (ret != 0) { + std::cerr << "backend::ListHeads " << ret << std::endl; + return ret; + } + for (const auto &head : heads) { + std::cout << head << std::endl; } + return 0; + } - assert(views.size() == 1u); - auto it = views.find(epoch); - assert(it != views.end()); + std::cerr << "usage:" << std::endl + << "zlog log ..." << std::endl + << "zlog link list" << std::endl + << "zlog head list" << std::endl; + return 1; +} - zlog_proto::View view_src; - if (!view_src.ParseFromString(it->second)) { - assert(0); - exit(1); - } +/* + * Handles log commands and returns an exit code. The accepted commands are + * - create + * - append + * - dump + * - trim + * - fill + * + * @param command the command to execute + * @param backend the backend to use + * @param filename the input filename for append commands + * + * @return exit code + */ +int handle_log(std::vector command, std::shared_ptr backend, std::string filename) { + const static std::map usages = { + { "create", "zlog log create " }, + { "append", "zlog log append \nzlog log append -i " }, + { "dump", "zlog log dump " }, + { "read", "zlog log read " }, + { "trim", "zlog log trim " }, + { "fill", "zlog log fill " }, + }; - auto view = std::make_shared(prefix, it->first, view_src); + if (command.size() > 0 && usages.find(command[0]) == usages.end()) { + std::cerr << "uknown command \"" << command[0] << "\"" << std::endl; + } - std::cout << "view@" << view->epoch() << std::endl; - for (auto it : view->object_map.stripes()) { - std::cout << " stripe@" << it.second.id() << " [" << it.first - << ", " << it.second.max_position() << "]" << std::endl; + if (command.size() == 1 && usages.find(command[0]) != usages.end()) { + std::cerr << "command requires log name" << std::endl; + } + + if (command.size() < 2 || usages.find(command[0]) == usages.end()) { + std::cerr << "usage:" << std::endl; + for (const auto &usage : usages) { + std::cerr << usage.second << std::endl; } + return 1; + } + + if (command[0] == "create") { + if (command.size() != 2) { // create + std::cerr << usages.at("create") << std::endl; + return 1; + } + std::string hoid, prefix; + int ret = backend->CreateLog(command[1], "", &hoid, &prefix); + switch (ret) { + case 0: + break; + case -EEXIST: + std::cerr << "error: log name already exists" << std::endl; + return ret; + case -EINVAL: + std::cerr << "error: invalid input" << std::endl; + return ret; + default: + std::cerr << "error: unknown error" << std::endl; + return ret; + } + std::cout << hoid << std::endl << prefix << std::endl; + return 0; + } + + // The rest of the commands need an opened log + zlog::Log *plog; + zlog::Options options; + options.backend = backend; + int ret = zlog::Log::Open(options, command[1], &plog); + switch (ret) { + case 0: + break; + case -ENOENT: + std::cerr << "error: no log named \"" + command[1] + "\"" << std::endl; + return ret; + case -EINVAL: + std::cerr << "error: invalid input" << std::endl; + default: + std::cerr << "error: unknown error " << ret << std::endl; + return ret; + } + std::unique_ptr log(plog); - epoch++; + if (command[0] == "append") { + uint64_t tail; + int ret = log->CheckTail(&tail); + if (ret != 0) { + std::cerr << "log::CheckTail " << ret << std::endl; + return ret; + } + if (command.size() == 2 && filename != "") { // append with input file + std::ifstream ifs(filename); + if (!ifs.is_open()) { + std::cerr << "no such file" << std::endl; + return 1; + } + std::string content( (std::istreambuf_iterator(ifs) ), + (std::istreambuf_iterator() ) ); + int ret = log->Append(content, &tail); + if (ret != 0) { + std::cerr << "log::Append " << ret << std::endl; + } + return ret; + } else if (command.size() == 3) { // append + int ret = log->Append(command[2], &tail); + if (ret != 0) { + std::cerr << "log::Append " << ret << std::endl; + } + return ret; + } else { + std::cerr << usages.at("append") << std::endl; + return 1; + } + } else if (command[0] == "dump") { + if (command.size() != 2) { // dump + std::cerr << usages.at("dump") << std::endl; + return 1; + } + uint64_t tail; + int ret = log->CheckTail(&tail); + if (ret != 0) { + std::cerr << "log::CheckTail " << ret << std::endl; + return ret; + } + std::string data; + for (uint64_t i = 0; i < tail; ++i) { + int ret = log->Read(i, &data); + switch (ret) { + case 0: + break; + case -ENODATA: + std::cerr << i << ": invalidated" << std::endl; + continue; + case -ERANGE: + std::cerr << i << ": free" << std::endl; + continue; + default: + std::cerr << "log::Read " << ret << std::endl; + return ret; + } + std::cout << i << ": "; + for (char c : data.substr(0, 80)) { + std::cout << std::setfill('0') << std::setw(2) << std::hex << static_cast(c); + } + std::cout << std::endl; + } + return 0; + } else if (command[0] == "read") { + if (command.size() != 3) { // read + std::cerr << usages.at("trim") << std::endl; + return 1; + } + uint64_t pos; + try { + pos = std::stoul(command[2]); + } catch (const std::invalid_argument &e) { + std::cerr << e.what() << std::endl; + return 1; + } + std::string data; + int ret = log->Read(pos, &data); + switch (ret) { + case 0: + break; + case -ENODATA: + std::cerr << "invalidated" << std::endl; + return ret; + case -ERANGE: + std::cerr << "free" << std::endl; + return ret; + default: + std::cerr << "log::Read " << ret << std::endl; + return ret; + } + for (char c : data) { + std::cout << std::setfill('0') << std::setw(2) << std::hex << static_cast(c); + } + std::cout << std::endl; + return 0; + } else if (command[0] == "trim") { + if (command.size() != 3) { // trim + std::cerr << usages.at("trim") << std::endl; + return 1; + } + uint64_t pos; + try { + pos = std::stoul(command[2]); + } catch (const std::invalid_argument &e) { + std::cerr << e.what() << std::endl; + return 1; + } + int ret = log->Trim(pos); + if (ret != 0) { + std::cerr << "log::Trim " << ret << std::endl; + } + return ret; + } else if (command[0] == "fill") { + if (command.size() != 3) { // fill + std::cerr << usages.at("fill") << std::endl; + return 1; + } + uint64_t pos; + try { + pos = std::stoul(command[2]); + } catch (const std::invalid_argument &e) { + std::cerr << e.what() << std::endl; + return 1; + } + int ret = log->Fill(pos); + if (ret != 0) { + std::cerr << "log::Fill " << ret << std::endl; + } + return ret; } - return 0; + // Should never reach here, but just to be safe + std::cerr << "usage:" << std::endl; + for (const auto &usage : usages) { + std::cerr << usage.second << std::endl; + } + return 1; }