From ae91d447df004142a208853572e74eabadbe380f Mon Sep 17 00:00:00 2001 From: Pavan C Date: Fri, 11 Feb 2022 03:08:01 -0700 Subject: [PATCH 1/8] Conv2d: Add custom batch size support --- src/conv2d_proj/headers/Conv2DSelect.h | 50 +++++++++++++++----------- src/tests/source/Conv2dProjTest.cc | 28 +++++++-------- 2 files changed, 42 insertions(+), 36 deletions(-) diff --git a/src/conv2d_proj/headers/Conv2DSelect.h b/src/conv2d_proj/headers/Conv2DSelect.h index 84c86173..22bc2f94 100644 --- a/src/conv2d_proj/headers/Conv2DSelect.h +++ b/src/conv2d_proj/headers/Conv2DSelect.h @@ -60,9 +60,10 @@ class Conv2DSelect: public SelectionComp{ } - Handle runEigenSpatial(TensorData& input, int z, int y, int x, int stride) { + Handle runEigenSpatial(TensorData& input, int n, int z, int y, int x, int stride) { - Eigen::TensorMap> a (input.rawData->c_ptr(), z, y, x); + std::cout << "---------------------------runEigenSpatial-------------------------:" << n << " " << z << " "<< y << " "<< x << " " << stride << std::endl; + Eigen::TensorMap> a (input.rawData->c_ptr(), n, z, y, x); //Eigen::Tensor c = a.convolve(b1) @@ -92,13 +93,13 @@ class Conv2DSelect: public SelectionComp{ //pre_contract_dims Eigen::array pre_contract_dims; pre_contract_dims[0] = zk * yk * xk; - pre_contract_dims[1] = (oy) * (ox); + pre_contract_dims[1] = (oy) * (ox) * x; //post_contract_dims Eigen::array post_contract_dims; post_contract_dims[0] = nk; post_contract_dims[1] = (oy); - post_contract_dims[2] = (ox); + post_contract_dims[2] = (ox) * x; //kernel dims Eigen::array kernel_dims; @@ -108,19 +109,20 @@ class Conv2DSelect: public SelectionComp{ //create the output - Handle> dimensions = makeObject>(3); - + Handle> dimensions = makeObject>(4); + dimensions->push_back(n); + dimensions->push_back(nk); dimensions->push_back(oy); dimensions->push_back(ox); - Handle out = makeObject(3, dimensions); + Handle out = makeObject(4, dimensions); - float * mempool = (float *) malloc (nk * oy * ox * sizeof(float)); + float * mempool = (float *) malloc (n * nk * oy * ox * sizeof(float)); - Eigen::TensorMap> c (mempool, nk, oy, ox); + Eigen::TensorMap> c (mempool, n, nk, oy, ox); c = b1.reshape(kernel_dims) .contract( @@ -139,15 +141,16 @@ class Conv2DSelect: public SelectionComp{ */ - memcpy (out->rawData->c_ptr(), mempool, nk * oy * ox * sizeof(float)); + memcpy (out->rawData->c_ptr(), mempool, n * nk * oy * ox * sizeof(float)); return out; } - Handle runAtenConv2d(TensorData& input, int z, int y, int x, int stride) { + Handle runAtenConv2d(TensorData& input, int n, int z, int y, int x, int stride) { + std::cout << "---------------------------RunAtendCOnv2d-------------------------:" << n << " " << z << " "<< y << " "<< x << " " << stride << std::endl; //input data - at::Tensor a = at::from_blob(input.rawData->c_ptr(), {1, z, y, x}); + at::Tensor a = at::from_blob(input.rawData->c_ptr(), {n, z, y, x}); at::Tensor b = at::from_blob(kernel->rawData->c_ptr(), {nk, zk, yk, xk}); @@ -159,7 +162,9 @@ class Conv2DSelect: public SelectionComp{ //create the output int oy = calculateOutputDimension(y, yk, stride); int ox = calculateOutputDimension(x, xk, stride); - Handle> dimensions = makeObject>(3); + Handle> dimensions = makeObject>(4); + + dimensions->push_back(n); dimensions->push_back(nk); @@ -167,8 +172,8 @@ class Conv2DSelect: public SelectionComp{ dimensions->push_back(ox); - Handle out = makeObject(3, dimensions); - memcpy(out->rawData->c_ptr(), c.storage().data(), nk * (oy) * (ox) * sizeof(float)); + Handle out = makeObject(4, dimensions); + memcpy(out->rawData->c_ptr(), c.storage().data(), n * nk * (oy) * (ox) * sizeof(float)); return out; } @@ -182,24 +187,27 @@ class Conv2DSelect: public SelectionComp{ TensorData input = *checkMe; - assert (input.numRanks = 3); + assert (input.numRanks = 4); //set up input dimensions + //N + int n = (*(input.dimensions))[0]; + //C - int z = (*(input.dimensions))[0]; + int z = (*(input.dimensions))[1]; //H - int y = (*(input.dimensions))[1]; + int y = (*(input.dimensions))[2]; //W - int x = (*(input.dimensions))[2]; + int x = (*(input.dimensions))[3]; if (conv2dMode == "eigen-spatial") { - return runEigenSpatial(input, z, y, x, stride); + return runEigenSpatial(input, n, z, y, x, stride); } else { - return runAtenConv2d(input, z, y, x, stride); + return runAtenConv2d(input, n, z, y, x, stride); } }); } diff --git a/src/tests/source/Conv2dProjTest.cc b/src/tests/source/Conv2dProjTest.cc index cdacb0ff..a2085039 100644 --- a/src/tests/source/Conv2dProjTest.cc +++ b/src/tests/source/Conv2dProjTest.cc @@ -11,12 +11,13 @@ using namespace std; -void load_rnd_img(int x, int y, int z, int size, pdb::PDBClient &pdbClient, +void load_rnd_img(int x, int y, int z, int n, int size, pdb::PDBClient &pdbClient, pdb::String dbName, pdb::String setName) { std::string errMsg; - pdb::makeObjectAllocatorBlock(64 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); pdb::Handle> dimensions = pdb::makeObject>(); + dimensions->push_back(n); dimensions->push_back(z); dimensions->push_back(y); dimensions->push_back(x); @@ -37,16 +38,12 @@ void load_rnd_img(int x, int y, int z, int size, pdb::PDBClient &pdbClient, for (int i = 0; i < size; i++) { try { - pdb::Handle image = pdb::makeObject(3, dimensions); - for (int c = 0; c < z; c++) { - for (int h = 0; h < y; h++) { - for (int w = 0; w < x; w++) { - double data = (bool)gen() ? distn(e2) : distp(e2); - (*(image->rawData))[c * y * x + h * x + w] = data; - } - } - } - images->push_back(image); + pdb::Handle image = pdb::makeObject(4, dimensions); + for (unsigned int i = 0; i < n*z*y*x; i++) { + double data = (bool)gen() ? distn(e2) : distp(e2); + (*(image->rawData))[i] = data; + } + images->push_back(image); } catch (pdb::NotEnoughSpace &e) { if (!pdbClient.sendData( pair(setName, dbName), images, errMsg)) { @@ -54,8 +51,9 @@ void load_rnd_img(int x, int y, int z, int size, pdb::PDBClient &pdbClient, exit(1); } i--; - pdb::makeObjectAllocatorBlock(64 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); dimensions = pdb::makeObject>(); + dimensions->push_back(n); dimensions->push_back(z); dimensions->push_back(y); dimensions->push_back(x); @@ -199,7 +197,7 @@ int main(int argc, char *argv[]) { //load data if (addDataOrNot == true) { - load_rnd_img(w, h, c, numImages, pdbClient, dbName, img_set); + load_rnd_img(w, h, c, n, numImages, pdbClient, dbName, img_set); } //create output dataset @@ -213,7 +211,7 @@ int main(int argc, char *argv[]) { } auto begin1 = std::chrono::high_resolution_clock::now(); - pdb::makeObjectAllocatorBlock(64 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); //create scan computation pdb::Handle imageScanner = From 3a2864b5168671334a631055cda336ca732c8b3a Mon Sep 17 00:00:00 2001 From: Pavan C Date: Mon, 21 Feb 2022 16:25:37 -0700 Subject: [PATCH 2/8] temp --- src/conv2d_proj/headers/Conv2DSelect.h | 12 +++++++++++- src/tests/source/Conv2dProjTest.cc | 7 ++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/conv2d_proj/headers/Conv2DSelect.h b/src/conv2d_proj/headers/Conv2DSelect.h index 22bc2f94..74092026 100644 --- a/src/conv2d_proj/headers/Conv2DSelect.h +++ b/src/conv2d_proj/headers/Conv2DSelect.h @@ -148,8 +148,9 @@ class Conv2DSelect: public SelectionComp{ Handle runAtenConv2d(TensorData& input, int n, int z, int y, int x, int stride) { - std::cout << "---------------------------RunAtendCOnv2d-------------------------:" << n << " " << z << " "<< y << " "<< x << " " << stride << std::endl; + std::cout << "---------------------------RunAtendConv2d-------------------------:" << n << " " << z << " "<< y << " "<< x << " " << stride << std::endl; //input data + // std::cout << "---------------------------input size ----------------------------:" << input.size << std::endl; at::Tensor a = at::from_blob(input.rawData->c_ptr(), {n, z, y, x}); at::Tensor b = at::from_blob(kernel->rawData->c_ptr(), {nk, zk, yk, xk}); @@ -157,7 +158,14 @@ class Conv2DSelect: public SelectionComp{ // bias length = kernel count = nk at::Tensor bias = at::zeros({nk}, at::kFloat); //perform the convolutional operation + auto begin = std::chrono::high_resolution_clock::now(); + // auto c = at::conv2d(a, b); auto c = at::conv2d(a, b, bias, stride); + auto end = std::chrono::high_resolution_clock::now(); + + std::cout << "-------------------------------------------------------------------------Inside RunAten Time Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; //create the output int oy = calculateOutputDimension(y, yk, stride); @@ -203,6 +211,8 @@ class Conv2DSelect: public SelectionComp{ //W int x = (*(input.dimensions))[3]; + std::cout << "------------------------------------------------------------------Before calling runAtenConv2d--------------------------------------------------" << std::endl; + std::cout << "n =" << n << "z =" << z << "y =" << y << "x =" << x << std::endl; if (conv2dMode == "eigen-spatial") { return runEigenSpatial(input, n, z, y, x, stride); diff --git a/src/tests/source/Conv2dProjTest.cc b/src/tests/source/Conv2dProjTest.cc index a2085039..9c7a63af 100644 --- a/src/tests/source/Conv2dProjTest.cc +++ b/src/tests/source/Conv2dProjTest.cc @@ -14,8 +14,9 @@ using namespace std; void load_rnd_img(int x, int y, int z, int n, int size, pdb::PDBClient &pdbClient, pdb::String dbName, pdb::String setName) { std::string errMsg; - pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); - + pdb::makeObjectAllocatorBlock(2047 * 1024 * 1024, true); + + std::cout << "----------------inside load rnd img---------------: " << x << " " << y << " " << z << " " << n << " " << size << std::endl; pdb::Handle> dimensions = pdb::makeObject>(); dimensions->push_back(n); dimensions->push_back(z); @@ -51,7 +52,7 @@ void load_rnd_img(int x, int y, int z, int n, int size, pdb::PDBClient &pdbClien exit(1); } i--; - pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(2047 * 1024 * 1024, true); dimensions = pdb::makeObject>(); dimensions->push_back(n); dimensions->push_back(z); From 64c88a6cde70b9d79e56ab7e20aa85054690e1e6 Mon Sep 17 00:00:00 2001 From: Pavan Date: Wed, 16 Feb 2022 21:18:17 -0700 Subject: [PATCH 3/8] Update README.md --- model-inference/convolutional-neural-network/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model-inference/convolutional-neural-network/README.md b/model-inference/convolutional-neural-network/README.md index 3f43d96b..aa00a87f 100644 --- a/model-inference/convolutional-neural-network/README.md +++ b/model-inference/convolutional-neural-network/README.md @@ -57,7 +57,7 @@ Once you start the netsDB system [Start Cluster](https://github.com/asu-cactus/n ### The version based on UDF-encapsulation ``` -bin/Conv2dProjTest whetherToLoadData whetherToCreateSet numImages conv2D-mode(aten-conv2d or eigen-spatial) inputDimension kernelDimension +bin/Conv2dProjTest whetherToLoadData whetherToCreateSet numImages conv2D-mode(aten-conv2d or eigen-spatial) inputDimension kernelDimension stride ``` You can pass the command line parameters or modify variables in the example application source to control the block shape and the size of the word2vec layer. From d463842cd0a8f64374bb22fa9022a380aa48a854 Mon Sep 17 00:00:00 2001 From: Pavan C Date: Sat, 26 Feb 2022 21:24:25 -0700 Subject: [PATCH 4/8] Temp Debugging --- src/conv2d_proj/headers/Conv2DSelect.h | 18 +- src/objectModel/headers/BuiltinPDBObjects.h | 276 ++++++++++---------- src/queries/headers/QueryClient.h | 3 +- src/tests/source/Conv2dProjTest.cc | 6 +- 4 files changed, 158 insertions(+), 145 deletions(-) diff --git a/src/conv2d_proj/headers/Conv2DSelect.h b/src/conv2d_proj/headers/Conv2DSelect.h index 74092026..71750e97 100644 --- a/src/conv2d_proj/headers/Conv2DSelect.h +++ b/src/conv2d_proj/headers/Conv2DSelect.h @@ -9,6 +9,7 @@ //LA libraries: #include #include +#include //ATen libraries: #include @@ -147,7 +148,7 @@ class Conv2DSelect: public SelectionComp{ } Handle runAtenConv2d(TensorData& input, int n, int z, int y, int x, int stride) { - + try { std::cout << "---------------------------RunAtendConv2d-------------------------:" << n << " " << z << " "<< y << " "<< x << " " << stride << std::endl; //input data // std::cout << "---------------------------input size ----------------------------:" << input.size << std::endl; @@ -155,18 +156,22 @@ class Conv2DSelect: public SelectionComp{ at::Tensor b = at::from_blob(kernel->rawData->c_ptr(), {nk, zk, yk, xk}); + ofstream myfile; + myfile.open ("conv2d_output_aten.txt"); // bias length = kernel count = nk at::Tensor bias = at::zeros({nk}, at::kFloat); //perform the convolutional operation auto begin = std::chrono::high_resolution_clock::now(); // auto c = at::conv2d(a, b); auto c = at::conv2d(a, b, bias, stride); + auto end = std::chrono::high_resolution_clock::now(); std::cout << "-------------------------------------------------------------------------Inside RunAten Time Duration: " << std::chrono::duration_cast>(end - begin).count() << " secs." << std::endl; + pdb::makeObjectAllocatorBlock(2047 * 1024 * 1024, true); //create the output int oy = calculateOutputDimension(y, yk, stride); int ox = calculateOutputDimension(x, xk, stride); @@ -181,9 +186,16 @@ class Conv2DSelect: public SelectionComp{ dimensions->push_back(ox); Handle out = makeObject(4, dimensions); - memcpy(out->rawData->c_ptr(), c.storage().data(), n * nk * (oy) * (ox) * sizeof(float)); - + memcpy(out->rawData->c_ptr(), c.storage().data(), (long long)n * nk * (oy) * (ox) * sizeof(float)); + // for (int i = 0; i < n * nk * oy * ox; i++) { + // myfile << (*(out->rawData))[i] << endl; + // } + // myfile.close(); return out; + } catch (std::exception &e) { + std::cerr << "-----------------------------exception caught--------------------------------------------------------: " << e.what() << '\n'; + exit(1); + } } diff --git a/src/objectModel/headers/BuiltinPDBObjects.h b/src/objectModel/headers/BuiltinPDBObjects.h index cd9077d2..a6195115 100644 --- a/src/objectModel/headers/BuiltinPDBObjects.h +++ b/src/objectModel/headers/BuiltinPDBObjects.h @@ -1,140 +1,138 @@ // Auto-generated by code in SConstruct -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/SetScan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StoragePagePinned.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/KMeansDoubleVector.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveDatabase.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/AggregationJobStage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageRemoveDatabase.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ComputePlan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageTestSetCopy.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ExecuteQuery.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ResourceInfo.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/NodeInfo.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageCollectStatsResponse.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSharedPage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatCreateDatabaseRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetSetRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddObjectInLoop.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PDBObjectPrototype.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ZB_Company.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Array.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/AggregationMap.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DepartmentEmployeeAges.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageCleanup.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Employee.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BroadcastJoinBuildHTJobStage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Avg.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetDatabaseRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetTypeResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddData.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/OptimizedDepartmentEmployees.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ExecuteComputation.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BuiltinPartialResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DepartmentEmployees.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/HashPartitionedJoinBuildHTJobStage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageRemoveTempSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddSharedMapping.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageCleanup.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveTempSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/JoinPairArray.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageNoMorePage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddTempSetResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageClearSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StoragePinPage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StringIntPair.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatPrintCatalogRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/JoinMap.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/GenericBlock.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DoubleSumResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageCollectStats.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CloseConnection.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueryPermitResponse.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSharedLibraryByNameRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DispatcherAddData.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddTempSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DeleteSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/OptimizedSupervisor.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSyncRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TopKQueue.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/OptimizedEmployee.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BaseQuery.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/SumResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/RequestResources.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatDeleteDatabaseRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/WriteUserSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageUnpinPage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/AvgResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Count.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DepartmentTotal.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatRegisterType.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DoubleVectorResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageGetData.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Holder.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StoragePinBytes.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetDatabaseResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ScanDoubleVectorSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageExportSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BackendTestSetCopy.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BackendTestSetScan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageGetSetPages.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddTempSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddSharedPage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageGetDataResponse.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/SetIdentifier.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatTypeNameSearchResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageExportSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Ack.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TupleSetExecuteQuery.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/MyEmployee.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddDatabase.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DoubleVector.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageBytesPinned.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatPrintCatalogResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageTestSetScan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageGetStats.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PairArray.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueryDone.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSyncResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ShutDown.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DispatcherRegisterPartitionPolicy.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageRemoveHashSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/SimpleRequestResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatDeleteSetRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueriesAndPlan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatalogUserTypeMetadata.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveHashSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetType.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddObject.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSetWithPartition.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageClearSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetSetResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/LambdaIdentifier.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DoneWithResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ListOfNodes.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ProcessorFactory.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSetObjectTypeRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddType.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageRemoveUserSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/KeepGoing.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PDBVector.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PDBMap.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/NodeDispatcherData.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Set.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ScanUserSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueryOutput.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PlaceOfQueryPlanner.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BackendExecuteSelection.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Supervisor.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Nothing.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSharedLibraryResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TensorBlockMeta.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSharedMapping.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddDatabase.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TupleSetJobStage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueryPermit.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/GetListOfNodes.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatCreateSetRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TensorBlockIdentifier.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Ack.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/AggregationJobStage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/AggregationMap.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Array.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Avg.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/AvgResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BackendExecuteSelection.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BackendTestSetCopy.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BackendTestSetScan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BaseQuery.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BroadcastJoinBuildHTJobStage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BuiltinPartialResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatalogUserTypeMetadata.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatCreateDatabaseRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatCreateSetRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatDeleteDatabaseRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatDeleteSetRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetDatabaseRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetDatabaseResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetSetRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetSetResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetType.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetTypeResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatPrintCatalogRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatPrintCatalogResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatRegisterType.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSetObjectTypeRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSharedLibraryByNameRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSharedLibraryResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSyncRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSyncResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatTypeNameSearchResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CloseConnection.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ComputePlan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Count.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DeleteSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DepartmentEmployeeAges.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DepartmentEmployees.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DepartmentTotal.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DispatcherAddData.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DispatcherRegisterPartitionPolicy.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddDatabase.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSetWithPartition.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSharedPage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddTempSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageCleanup.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageClearSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageExportSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveDatabase.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveHashSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveTempSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DoneWithResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DoubleSumResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DoubleVector.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DoubleVectorResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Employee.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ExecuteComputation.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ExecuteQuery.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/GenericBlock.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/GetListOfNodes.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/HashPartitionedJoinBuildHTJobStage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Holder.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/JoinMap.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/JoinPairArray.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/KeepGoing.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/KMeansDoubleVector.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/LambdaIdentifier.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ListOfNodes.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/MyEmployee.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/NodeDispatcherData.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/NodeInfo.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Nothing.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/OptimizedDepartmentEmployees.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/OptimizedEmployee.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/OptimizedSupervisor.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PairArray.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PDBMap.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PDBObjectPrototype.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PDBVector.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PlaceOfQueryPlanner.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ProcessorFactory.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueriesAndPlan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueryDone.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueryOutput.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueryPermit.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueryPermitResponse.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/RequestResources.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ResourceInfo.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ScanDoubleVectorSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ScanUserSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Set.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/SetIdentifier.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/SetScan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ShutDown.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/SimpleRequestResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddData.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddDatabase.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddObject.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddObjectInLoop.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddSharedPage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddTempSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddTempSetResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddType.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageBytesPinned.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageCleanup.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageClearSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageCollectStats.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageCollectStatsResponse.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageExportSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageGetData.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageGetDataResponse.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageGetSetPages.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageGetStats.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageNoMorePage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StoragePagePinned.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StoragePinBytes.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StoragePinPage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageRemoveDatabase.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageRemoveHashSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageRemoveTempSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageRemoveUserSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageTestSetCopy.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageTestSetScan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageUnpinPage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StringIntPair.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/SumResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Supervisor.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TensorBlockIdentifier.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TensorBlockMeta.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TopKQueue.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TupleSetExecuteQuery.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TupleSetJobStage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/WriteUserSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ZB_Company.h" diff --git a/src/queries/headers/QueryClient.h b/src/queries/headers/QueryClient.h index 09ae978c..a31fb5b4 100644 --- a/src/queries/headers/QueryClient.h +++ b/src/queries/headers/QueryClient.h @@ -160,6 +160,7 @@ if (typeName != getTypeName ()) { bool executeComputations(std::string& errMsg, std::string jobName = "", bool preCompile = false) { // this is the request + std::cout << "------------------------------------------------------------------------------------------------inside execute computations----------------------------" << std::endl; const UseTemporaryAllocationBlock myBlock{256 * 1024 * 1024}; QueryGraphAnalyzer queryAnalyzer(this->queryGraph); std::string tcapString = queryAnalyzer.parseTCAPString(); @@ -175,7 +176,7 @@ if (typeName != getTypeName ()) { jobName = clientName + "-" + std::to_string(queryId); queryId ++; } - std::cout << "jobName is " << jobName << std::endl; + std::cout << "-----------------------------------------jobName is --------------------------------------------------------------------" << jobName << std::endl; Handle executeComputation = makeObject( jobName, tcapString, preCompile); diff --git a/src/tests/source/Conv2dProjTest.cc b/src/tests/source/Conv2dProjTest.cc index 9c7a63af..983458dd 100644 --- a/src/tests/source/Conv2dProjTest.cc +++ b/src/tests/source/Conv2dProjTest.cc @@ -42,7 +42,7 @@ void load_rnd_img(int x, int y, int z, int n, int size, pdb::PDBClient &pdbClien pdb::Handle image = pdb::makeObject(4, dimensions); for (unsigned int i = 0; i < n*z*y*x; i++) { double data = (bool)gen() ? distn(e2) : distp(e2); - (*(image->rawData))[i] = data; + (*(image->rawData))[i] = 1; } images->push_back(image); } catch (pdb::NotEnoughSpace &e) { @@ -212,7 +212,7 @@ int main(int argc, char *argv[]) { } auto begin1 = std::chrono::high_resolution_clock::now(); - pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(2047 * 1024 * 1024, true); //create scan computation pdb::Handle imageScanner = @@ -248,6 +248,8 @@ int main(int argc, char *argv[]) { auto begin = std::chrono::high_resolution_clock::now(); + + std::cout<< "---------------------------Starting execute computations---------------------------" << std::endl; // run the computation if (!pdbClient.executeComputations(errMsg, "conv2d-proj", myWriteSet)) { cout << "Computation failed. Message was: " << errMsg << "\n"; From dcc6c3dace7dc10e9e3d39144e89d374f764b6e5 Mon Sep 17 00:00:00 2001 From: Pavan Chikkathimmegowda Date: Tue, 1 Mar 2022 20:20:54 -0700 Subject: [PATCH 5/8] Customize page size --- src/conf/headers/Configuration.h | 2 +- src/conv2d_proj/headers/Conv2DSelect.h | 5 ++++- src/tests/source/Conv2dProjTest.cc | 10 +++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/conf/headers/Configuration.h b/src/conf/headers/Configuration.h index 5d21d53f..6fc03c8d 100644 --- a/src/conf/headers/Configuration.h +++ b/src/conf/headers/Configuration.h @@ -24,7 +24,7 @@ using namespace std; #endif #ifndef DEFAULT_MAX_PAGE_SIZE -#define DEFAULT_MAX_PAGE_SIZE ((size_t)(1024) * (size_t)(1024) * (size_t)(1024)) +#define DEFAULT_MAX_PAGE_SIZE ((size_t)(2047) * (size_t)(1024) * (size_t)(1024)) #endif #ifndef DEFAULT_NET_PAGE_SIZE diff --git a/src/conv2d_proj/headers/Conv2DSelect.h b/src/conv2d_proj/headers/Conv2DSelect.h index 71750e97..249c4a49 100644 --- a/src/conv2d_proj/headers/Conv2DSelect.h +++ b/src/conv2d_proj/headers/Conv2DSelect.h @@ -171,7 +171,7 @@ class Conv2DSelect: public SelectionComp{ << std::chrono::duration_cast>(end - begin).count() << " secs." << std::endl; - pdb::makeObjectAllocatorBlock(2047 * 1024 * 1024, true); + // pdb::makeObjectAllocatorBlock(2047 * 1024 * 1024, true); //create the output int oy = calculateOutputDimension(y, yk, stride); int ox = calculateOutputDimension(x, xk, stride); @@ -192,6 +192,9 @@ class Conv2DSelect: public SelectionComp{ // } // myfile.close(); return out; + } catch (NotEnoughSpace &e) { + std::cout<<"------------------inside not enough space allocation-----------------:" << '\n'; + exit(1); } catch (std::exception &e) { std::cerr << "-----------------------------exception caught--------------------------------------------------------: " << e.what() << '\n'; exit(1); diff --git a/src/tests/source/Conv2dProjTest.cc b/src/tests/source/Conv2dProjTest.cc index 983458dd..ff922212 100644 --- a/src/tests/source/Conv2dProjTest.cc +++ b/src/tests/source/Conv2dProjTest.cc @@ -14,7 +14,7 @@ using namespace std; void load_rnd_img(int x, int y, int z, int n, int size, pdb::PDBClient &pdbClient, pdb::String dbName, pdb::String setName) { std::string errMsg; - pdb::makeObjectAllocatorBlock(2047 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(2000 * 1024 * 1024, true); std::cout << "----------------inside load rnd img---------------: " << x << " " << y << " " << z << " " << n << " " << size << std::endl; pdb::Handle> dimensions = pdb::makeObject>(); @@ -52,7 +52,7 @@ void load_rnd_img(int x, int y, int z, int n, int size, pdb::PDBClient &pdbClien exit(1); } i--; - pdb::makeObjectAllocatorBlock(2047 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(2000 * 1024 * 1024, true); dimensions = pdb::makeObject>(); dimensions->push_back(n); dimensions->push_back(z); @@ -189,7 +189,7 @@ int main(int argc, char *argv[]) { pdbClient.removeSet(dbName, img_set, errMsg); if (!pdbClient.createSet( - dbName, img_set, errMsg, (size_t)128 * (size_t)1024 * (size_t)1024)) { + dbName, img_set, errMsg, (size_t)1024 * (size_t)1024 * (size_t)1024)) { cout << "Not able to create set " + img_set + ": " + errMsg; } else { cout << "Created set " << img_set << ".\n"; @@ -205,14 +205,14 @@ int main(int argc, char *argv[]) { string feature_out_set = "feature_map"; pdbClient.removeSet(dbName, feature_out_set, errMsg); if (!pdbClient.createSet( - dbName, feature_out_set, errMsg, (size_t)128 * (size_t)1024 * (size_t)1024)) { + dbName, feature_out_set, errMsg, (size_t)2047 * (size_t)1024 * (size_t)1024)) { cout << "Not able to create set " + feature_out_set + ": " + errMsg; } else { cout << "Created set " << feature_out_set << ".\n"; } auto begin1 = std::chrono::high_resolution_clock::now(); - pdb::makeObjectAllocatorBlock(2047 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(1024 * 1024 * 1024, true); //create scan computation pdb::Handle imageScanner = From 2dfcdc7c724dd785bf6cd7a98a0f65b4611f3ba0 Mon Sep 17 00:00:00 2001 From: Pavan Chikkathimmegowda Date: Thu, 10 Mar 2022 01:01:15 -0700 Subject: [PATCH 6/8] 40, 64, 112, 112 --- src/conv2d_proj/headers/Conv2DSelect.h | 13 ++++++++----- src/tests/source/Conv2dProjTest.cc | 16 ++++++++-------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/conv2d_proj/headers/Conv2DSelect.h b/src/conv2d_proj/headers/Conv2DSelect.h index 249c4a49..030f6ce8 100644 --- a/src/conv2d_proj/headers/Conv2DSelect.h +++ b/src/conv2d_proj/headers/Conv2DSelect.h @@ -156,9 +156,7 @@ class Conv2DSelect: public SelectionComp{ at::Tensor b = at::from_blob(kernel->rawData->c_ptr(), {nk, zk, yk, xk}); - ofstream myfile; - myfile.open ("conv2d_output_aten.txt"); - // bias length = kernel count = nk + // bias length = kernel Count = nk at::Tensor bias = at::zeros({nk}, at::kFloat); //perform the convolutional operation auto begin = std::chrono::high_resolution_clock::now(); @@ -187,6 +185,11 @@ class Conv2DSelect: public SelectionComp{ Handle out = makeObject(4, dimensions); memcpy(out->rawData->c_ptr(), c.storage().data(), (long long)n * nk * (oy) * (ox) * sizeof(float)); + + // Write output to file + // ofstream myfile; + // std::time_t result = std::time(nullptr); + // myfile.open ("conv2d_output_aten" + to_string(result) + ".txt"); // for (int i = 0; i < n * nk * oy * ox; i++) { // myfile << (*(out->rawData))[i] << endl; // } @@ -259,8 +262,8 @@ class Conv2DSelect: public SelectionComp{ unsigned int stride; - static int calculateOutputDimension(int inputDimention, int filterDimention, int stride) { - return (inputDimention - filterDimention) / stride + 1; + static int calculateOutputDimension(int inputDimension, int filterDimension, int stride) { + return (inputDimension - filterDimension) / stride + 1; } }; diff --git a/src/tests/source/Conv2dProjTest.cc b/src/tests/source/Conv2dProjTest.cc index ff922212..12232023 100644 --- a/src/tests/source/Conv2dProjTest.cc +++ b/src/tests/source/Conv2dProjTest.cc @@ -14,7 +14,7 @@ using namespace std; void load_rnd_img(int x, int y, int z, int n, int size, pdb::PDBClient &pdbClient, pdb::String dbName, pdb::String setName) { std::string errMsg; - pdb::makeObjectAllocatorBlock(2000 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); std::cout << "----------------inside load rnd img---------------: " << x << " " << y << " " << z << " " << n << " " << size << std::endl; pdb::Handle> dimensions = pdb::makeObject>(); @@ -41,8 +41,8 @@ void load_rnd_img(int x, int y, int z, int n, int size, pdb::PDBClient &pdbClien try { pdb::Handle image = pdb::makeObject(4, dimensions); for (unsigned int i = 0; i < n*z*y*x; i++) { - double data = (bool)gen() ? distn(e2) : distp(e2); - (*(image->rawData))[i] = 1; + float data = (bool)gen() ? distn(e2) : distp(e2); + (*(image->rawData))[i] = data; } images->push_back(image); } catch (pdb::NotEnoughSpace &e) { @@ -52,7 +52,7 @@ void load_rnd_img(int x, int y, int z, int n, int size, pdb::PDBClient &pdbClien exit(1); } i--; - pdb::makeObjectAllocatorBlock(2000 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); dimensions = pdb::makeObject>(); dimensions->push_back(n); dimensions->push_back(z); @@ -189,7 +189,7 @@ int main(int argc, char *argv[]) { pdbClient.removeSet(dbName, img_set, errMsg); if (!pdbClient.createSet( - dbName, img_set, errMsg, (size_t)1024 * (size_t)1024 * (size_t)1024)) { + dbName, img_set, errMsg, (size_t)512 * (size_t)1024 * (size_t)1024)) { cout << "Not able to create set " + img_set + ": " + errMsg; } else { cout << "Created set " << img_set << ".\n"; @@ -205,14 +205,14 @@ int main(int argc, char *argv[]) { string feature_out_set = "feature_map"; pdbClient.removeSet(dbName, feature_out_set, errMsg); if (!pdbClient.createSet( - dbName, feature_out_set, errMsg, (size_t)2047 * (size_t)1024 * (size_t)1024)) { + dbName, feature_out_set, errMsg, (size_t)1024 * (size_t)1024 * (size_t)1024)) { cout << "Not able to create set " + feature_out_set + ": " + errMsg; } else { cout << "Created set " << feature_out_set << ".\n"; } auto begin1 = std::chrono::high_resolution_clock::now(); - pdb::makeObjectAllocatorBlock(1024 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(8 * 1024 * 1024, true); //create scan computation pdb::Handle imageScanner = @@ -254,7 +254,7 @@ int main(int argc, char *argv[]) { if (!pdbClient.executeComputations(errMsg, "conv2d-proj", myWriteSet)) { cout << "Computation failed. Message was: " << errMsg << "\n"; exit(1); - } + } auto end = std::chrono::high_resolution_clock::now(); From a45a3b2ee87f091364c3bef524923d4a6133bfdd Mon Sep 17 00:00:00 2001 From: Pavan Chikkathimmegowda Date: Sat, 26 Mar 2022 00:44:12 -0700 Subject: [PATCH 7/8] Query client log --- src/queries/headers/QueryClient.h | 38 ++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/queries/headers/QueryClient.h b/src/queries/headers/QueryClient.h index a31fb5b4..f1a637c6 100644 --- a/src/queries/headers/QueryClient.h +++ b/src/queries/headers/QueryClient.h @@ -162,29 +162,62 @@ if (typeName != getTypeName ()) { // this is the request std::cout << "------------------------------------------------------------------------------------------------inside execute computations----------------------------" << std::endl; const UseTemporaryAllocationBlock myBlock{256 * 1024 * 1024}; + auto begin = std::chrono::high_resolution_clock::now(); QueryGraphAnalyzer queryAnalyzer(this->queryGraph); + auto end = std::chrono::high_resolution_clock::now(); + std::cout << "Query Analyzer Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; + + begin = std::chrono::high_resolution_clock::now(); std::string tcapString = queryAnalyzer.parseTCAPString(); + end = std::chrono::high_resolution_clock::now(); + std::cout << "Query Parser Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; + + + begin = std::chrono::high_resolution_clock::now(); std::vector> computations; queryAnalyzer.parseComputations(computations); + end = std::chrono::high_resolution_clock::now(); + std::cout << "parseComputations Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; + + begin = std::chrono::high_resolution_clock::now(); Handle>> computationsToSend = makeObject>>(); for (int i = 0; i < computations.size(); i++) { computationsToSend->push_back(computations[i]); } + end = std::chrono::high_resolution_clock::now(); + std::cout << "computationsToSend Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; if (jobName == "") { jobName = clientName + "-" + std::to_string(queryId); queryId ++; } std::cout << "-----------------------------------------jobName is --------------------------------------------------------------------" << jobName << std::endl; + std::cout << "jobName is " << jobName << std::endl; + + begin = std::chrono::high_resolution_clock::now(); Handle executeComputation = makeObject( jobName, tcapString, preCompile); + end = std::chrono::high_resolution_clock::now(); + std::cout << "executeComputation Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; // this call asks the database to execute the query, and then it inserts the result set name // within each of the results, as well as the database connection information // this is for query scheduling stuff if (useScheduler == true) { + cout << "-----------------------------------------Inside useScheduler-----------------------------------------" << std::endl; + begin = std::chrono::high_resolution_clock::now(); return simpleDoubleRequest>, SimpleRequestResult, @@ -213,7 +246,10 @@ if (typeName != getTypeName ()) { executeComputation, computationsToSend); - + end = std::chrono::high_resolution_clock::now(); + std::cout << "useScheduler Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; } else { errMsg = "This query must be sent to QuerySchedulerServer, but it seems " From bcc3fda1a05aa621de6fe92316d64b841c7895b9 Mon Sep 17 00:00:00 2001 From: Pavan Chikkathimmegowda Date: Fri, 8 Apr 2022 19:34:05 -0700 Subject: [PATCH 8/8] run pipeline time log --- src/queryExecution/source/PipelineStage.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/queryExecution/source/PipelineStage.cc b/src/queryExecution/source/PipelineStage.cc index 0700504f..d93fc8d1 100644 --- a/src/queryExecution/source/PipelineStage.cc +++ b/src/queryExecution/source/PipelineStage.cc @@ -907,9 +907,14 @@ void PipelineStage::executePipelineWork(int i, << "-" << this->jobStage->getTargetComputationSpecifier() << std::chrono::duration_cast>(end - begin).count() << " secs." << std::endl; + auto start = std::chrono::high_resolution_clock::now(); std::cout << i<<": Running Pipeline\n"; curPipeline->id = i; curPipeline->run(); + end = std::chrono::high_resolution_clock::now(); + std::cout << "RunPipeline Duration : " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; curPipeline = nullptr; newPlan->nullifyPlanPointer(); getAllocator().setPolicy(AllocatorPolicy::defaultAllocator);