diff --git a/sql/tbam_tbl_ddl.sql b/sql/tbam_tbl_ddl.sql index 1afa46d..d40d7a0 100644 --- a/sql/tbam_tbl_ddl.sql +++ b/sql/tbam_tbl_ddl.sql @@ -24,7 +24,7 @@ CREATE TABLE LINEITEM ( L_SHIPMODE TEXT, L_COMMENT TEXT, primary key(L_ORDERKEY, L_LINENUMBER) -) USING kv_am; +) USING kv_am WITH (storage=column,batch_capacity=307200); -- Insert example -- -- INSERT INTO LINEITEM VALUES ((5086273,1),182032,4551,21,23394.63,0.02,0.07,'A','F','1992-12-07','1992-12-16','1992-12-18','TAKE BACK RETURN','RAIL','s. furiously special packages detect blithe'); diff --git a/src/benchmark.cc b/src/benchmark.cc index f78c0ba..484e306 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -48,13 +48,15 @@ int main(int argc, char** argv) { if (StringEquals(scenario, kInsert)) { s->BenchInsertScenario(); } else if (StringEquals(scenario, kLoad)) { - s->BenchLoadScenario(); + s->BenchLoadScenario(); } else if (StringEquals(scenario, kScan)) { s->BenchScanScenario(); } else if (StringEquals(scenario, kGetRandom)) { s->BenchGetScenario(GetType::GetRand); } else if (StringEquals(scenario, kGetLast)) { s->BenchGetScenario(GetType::GetLast); + } else if (StringEquals(scenario, kRangeQuery)) { + s->BenchRangeQueryScenario(); } else { cout << "unsupported scenario!" << endl; } diff --git a/src/benchmark.h b/src/benchmark.h index 5435a22..4e534cf 100644 --- a/src/benchmark.h +++ b/src/benchmark.h @@ -37,9 +37,11 @@ const string kLoad = "load"; const string kScan = "scan"; const string kGetRandom = "getrand"; const string kGetLast = "getlast"; +const string kRangeQuery = "rangequery"; const int kWarmCount = 0; const int kGetCount = 10000; +const int kColumnCount = 14; const string delim = "|"; enum GetType { @@ -60,6 +62,9 @@ class BenchmarkScenario { virtual void BenchLoadScenario(void* args = nullptr) {} virtual void BenchScanScenario(void* args = nullptr) {} virtual void BenchGetScenario(GetType type) {} + virtual void BenchRangeQueryScenario(void* args = nullptr) { + cout << "Only engine platform support range query ..." << endl; + }; virtual bool PrepareBenchmarkData() { return true; } virtual void DisplayBenchmarkInfo() {} diff --git a/src/eng_benchmark.h b/src/eng_benchmark.h index c9ea4c1..215af99 100755 --- a/src/eng_benchmark.h +++ b/src/eng_benchmark.h @@ -10,10 +10,15 @@ using namespace std; #include "vidardb/options.h" #include "vidardb/table.h" #include "vidardb/cache.h" +#include "vidardb/file_iter.h" +#include "vidardb/comparator.h" +#include "vidardb/splitter.h" #include "benchmark.h" #include "util.h" using namespace vidardb; +#define BUFSIZE 300 * 1024 * 1024 // 300MB + void PutFixed32(string* dst, uint32_t value); class EngBenchmarkScenario : public BenchmarkScenario { @@ -26,7 +31,8 @@ class EngBenchmarkScenario : public BenchmarkScenario { virtual void BenchLoadScenario(void* args = nullptr) override; virtual void BenchScanScenario(void* args = nullptr) override; virtual void BenchGetScenario(GetType type) override; - + virtual void BenchRangeQueryScenario(void* args = nullptr) override; + virtual bool PrepareBenchmarkData() override; virtual void DisplayBenchmarkInfo() override; @@ -51,13 +57,36 @@ bool EngBenchmarkScenario::BeforeBenchmark(void* args) { // options.IncreaseParallelism(); // options.OptimizeLevelStyleCompaction(); // options.PrepareForBulkLoad(); + BlockBasedTableOptions block_based_options; block_based_options.block_cache = NewLRUCache(static_cast(96 * 1024 * 1024)); - options.table_factory.reset(NewBlockBasedTableFactory(block_based_options)); + shared_ptr block_based_table( + NewBlockBasedTableFactory(block_based_options)); + + ColumnTableOptions column_table_options; + column_table_options.block_cache = + NewLRUCache(static_cast(96 * 1024 * 1024)); + column_table_options.column_count = kColumnCount; + for (int i = 0; i < column_table_options.column_count; i++) { + column_table_options.value_comparators.push_back(BytewiseComparator()); + } + shared_ptr column_table( + NewColumnTableFactory(column_table_options)); + + string scenario(getenv(kScenario)); + bool use_column = StringEquals(scenario, kRangeQuery); + + options.splitter.reset(NewPipeSplitter()); options.OptimizeAdaptiveLevelStyleCompaction(128*1024*1024); + options.table_factory.reset(NewAdaptiveTableFactory(block_based_table, + block_based_table, column_table, use_column? 0: -1)); Status s = DB::Open(options, dbpath, &db); + if (!s.ok()) { + cout << s.ToString() << endl; + } + return s.ok(); } @@ -181,6 +210,77 @@ void EngBenchmarkScenario::BenchScanScenario(void* args) { << seconds << " s, tps = " << tps << endl; } +void EngBenchmarkScenario::BenchRangeQueryScenario(void* args) { + if (!PrepareBenchmarkData()) { + cout << "Prepare data failed" << endl; + return; + } + db->CompactRange(CompactRangeOptions(), nullptr, nullptr); + + string value; + ReadOptions ro; + // ro.columns.push_back(6); // tax + unsigned long long count = 0, index = 1; + FileIter *it = static_cast(db->NewFileIterator(ro)); + vector block_bits; // full scan + + cout << "Start to benchmark range query rate ..." << endl; + auto start = chrono::high_resolution_clock::now(); + + for (it->SeekToFirst(); it->Valid(); it->Next(), index++) { + vector> min_max; + auto mix_max_start = chrono::high_resolution_clock::now(); + Status s = it->GetMinMax(min_max); + if (s.IsNotFound()) { + continue; + } + if (!s.ok()) { + cout << s.ToString() << endl; + continue; + } + assert(s.ok()); + + auto min_max_end = chrono::high_resolution_clock::now(); + chrono::duration min_max_ms = + min_max_end - mix_max_start; + cout << "MinMax" << index << ": " << min_max_ms.count() << " ms" << endl; + + uint64_t valid_count, total_count; + char* buf = new char[BUFSIZE]; + + auto range_query_start = chrono::high_resolution_clock::now(); + s = it->RangeQuery(block_bits, buf, BUFSIZE, &valid_count, &total_count); + assert(s.ok()); + auto range_query_end = chrono::high_resolution_clock::now(); + chrono::duration range_query_ms = + range_query_end - range_query_start; + cout << "RangeQuery" << index << ": " << valid_count << ": " + << total_count << ": " << range_query_ms.count() << " ms" + << endl; + + // uint64_t* end = reinterpret_cast(buf + BUFSIZ); + // for (auto c : ro.columns) { + // for (int i = 0; i < count; ++i) { + // uint64_t offset = *(--end), size = *(--end); + // value.assign(buf + offset, size); + // DecodeTuple(value); + // } + // } + + delete buf; + count += valid_count; + } + + auto end = chrono::high_resolution_clock::now(); + delete it; + + chrono::duration ms = end - start; + double seconds = ms.count() / 1000; + double tps = count / seconds; + cout << "Range query " << count << " rows and take " + << seconds << " s, tps = " << tps << endl; +} + void EngBenchmarkScenario::Get(int n, GetType type) { vector> v; PrepareGetData(v, type, n);