diff --git a/model-inference/convolutional-neural-network/README.md b/model-inference/convolutional-neural-network/README.md index 3f43d96b..aa00a87f 100644 --- a/model-inference/convolutional-neural-network/README.md +++ b/model-inference/convolutional-neural-network/README.md @@ -57,7 +57,7 @@ Once you start the netsDB system [Start Cluster](https://github.com/asu-cactus/n ### The version based on UDF-encapsulation ``` -bin/Conv2dProjTest whetherToLoadData whetherToCreateSet numImages conv2D-mode(aten-conv2d or eigen-spatial) inputDimension kernelDimension +bin/Conv2dProjTest whetherToLoadData whetherToCreateSet numImages conv2D-mode(aten-conv2d or eigen-spatial) inputDimension kernelDimension stride ``` You can pass the command line parameters or modify variables in the example application source to control the block shape and the size of the word2vec layer. diff --git a/src/conf/headers/Configuration.h b/src/conf/headers/Configuration.h index 5d21d53f..6fc03c8d 100644 --- a/src/conf/headers/Configuration.h +++ b/src/conf/headers/Configuration.h @@ -24,7 +24,7 @@ using namespace std; #endif #ifndef DEFAULT_MAX_PAGE_SIZE -#define DEFAULT_MAX_PAGE_SIZE ((size_t)(1024) * (size_t)(1024) * (size_t)(1024)) +#define DEFAULT_MAX_PAGE_SIZE ((size_t)(2047) * (size_t)(1024) * (size_t)(1024)) #endif #ifndef DEFAULT_NET_PAGE_SIZE diff --git a/src/conv2d_proj/headers/Conv2DSelect.h b/src/conv2d_proj/headers/Conv2DSelect.h index 84c86173..030f6ce8 100644 --- a/src/conv2d_proj/headers/Conv2DSelect.h +++ b/src/conv2d_proj/headers/Conv2DSelect.h @@ -9,6 +9,7 @@ //LA libraries: #include #include +#include //ATen libraries: #include @@ -60,9 +61,10 @@ class Conv2DSelect: public SelectionComp{ } - Handle runEigenSpatial(TensorData& input, int z, int y, int x, int stride) { + Handle runEigenSpatial(TensorData& input, int n, int z, int y, int x, int stride) { - Eigen::TensorMap> a (input.rawData->c_ptr(), z, y, x); + std::cout << "---------------------------runEigenSpatial-------------------------:" << n << " " << z << " "<< y << " "<< x << " " << stride << std::endl; + Eigen::TensorMap> a (input.rawData->c_ptr(), n, z, y, x); //Eigen::Tensor c = a.convolve(b1) @@ -92,13 +94,13 @@ class Conv2DSelect: public SelectionComp{ //pre_contract_dims Eigen::array pre_contract_dims; pre_contract_dims[0] = zk * yk * xk; - pre_contract_dims[1] = (oy) * (ox); + pre_contract_dims[1] = (oy) * (ox) * x; //post_contract_dims Eigen::array post_contract_dims; post_contract_dims[0] = nk; post_contract_dims[1] = (oy); - post_contract_dims[2] = (ox); + post_contract_dims[2] = (ox) * x; //kernel dims Eigen::array kernel_dims; @@ -108,19 +110,20 @@ class Conv2DSelect: public SelectionComp{ //create the output - Handle> dimensions = makeObject>(3); - + Handle> dimensions = makeObject>(4); + dimensions->push_back(n); + dimensions->push_back(nk); dimensions->push_back(oy); dimensions->push_back(ox); - Handle out = makeObject(3, dimensions); + Handle out = makeObject(4, dimensions); - float * mempool = (float *) malloc (nk * oy * ox * sizeof(float)); + float * mempool = (float *) malloc (n * nk * oy * ox * sizeof(float)); - Eigen::TensorMap> c (mempool, nk, oy, ox); + Eigen::TensorMap> c (mempool, n, nk, oy, ox); c = b1.reshape(kernel_dims) .contract( @@ -139,27 +142,40 @@ class Conv2DSelect: public SelectionComp{ */ - memcpy (out->rawData->c_ptr(), mempool, nk * oy * ox * sizeof(float)); + memcpy (out->rawData->c_ptr(), mempool, n * nk * oy * ox * sizeof(float)); return out; } - Handle runAtenConv2d(TensorData& input, int z, int y, int x, int stride) { - + Handle runAtenConv2d(TensorData& input, int n, int z, int y, int x, int stride) { + try { + std::cout << "---------------------------RunAtendConv2d-------------------------:" << n << " " << z << " "<< y << " "<< x << " " << stride << std::endl; //input data - at::Tensor a = at::from_blob(input.rawData->c_ptr(), {1, z, y, x}); + // std::cout << "---------------------------input size ----------------------------:" << input.size << std::endl; + at::Tensor a = at::from_blob(input.rawData->c_ptr(), {n, z, y, x}); at::Tensor b = at::from_blob(kernel->rawData->c_ptr(), {nk, zk, yk, xk}); - // bias length = kernel count = nk + // bias length = kernel Count = nk at::Tensor bias = at::zeros({nk}, at::kFloat); //perform the convolutional operation + auto begin = std::chrono::high_resolution_clock::now(); + // auto c = at::conv2d(a, b); auto c = at::conv2d(a, b, bias, stride); + auto end = std::chrono::high_resolution_clock::now(); + + std::cout << "-------------------------------------------------------------------------Inside RunAten Time Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; + + // pdb::makeObjectAllocatorBlock(2047 * 1024 * 1024, true); //create the output int oy = calculateOutputDimension(y, yk, stride); int ox = calculateOutputDimension(x, xk, stride); - Handle> dimensions = makeObject>(3); + Handle> dimensions = makeObject>(4); + + dimensions->push_back(n); dimensions->push_back(nk); @@ -167,10 +183,25 @@ class Conv2DSelect: public SelectionComp{ dimensions->push_back(ox); - Handle out = makeObject(3, dimensions); - memcpy(out->rawData->c_ptr(), c.storage().data(), nk * (oy) * (ox) * sizeof(float)); + Handle out = makeObject(4, dimensions); + memcpy(out->rawData->c_ptr(), c.storage().data(), (long long)n * nk * (oy) * (ox) * sizeof(float)); + // Write output to file + // ofstream myfile; + // std::time_t result = std::time(nullptr); + // myfile.open ("conv2d_output_aten" + to_string(result) + ".txt"); + // for (int i = 0; i < n * nk * oy * ox; i++) { + // myfile << (*(out->rawData))[i] << endl; + // } + // myfile.close(); return out; + } catch (NotEnoughSpace &e) { + std::cout<<"------------------inside not enough space allocation-----------------:" << '\n'; + exit(1); + } catch (std::exception &e) { + std::cerr << "-----------------------------exception caught--------------------------------------------------------: " << e.what() << '\n'; + exit(1); + } } @@ -182,24 +213,29 @@ class Conv2DSelect: public SelectionComp{ TensorData input = *checkMe; - assert (input.numRanks = 3); + assert (input.numRanks = 4); //set up input dimensions + //N + int n = (*(input.dimensions))[0]; + //C - int z = (*(input.dimensions))[0]; + int z = (*(input.dimensions))[1]; //H - int y = (*(input.dimensions))[1]; + int y = (*(input.dimensions))[2]; //W - int x = (*(input.dimensions))[2]; + int x = (*(input.dimensions))[3]; + std::cout << "------------------------------------------------------------------Before calling runAtenConv2d--------------------------------------------------" << std::endl; + std::cout << "n =" << n << "z =" << z << "y =" << y << "x =" << x << std::endl; if (conv2dMode == "eigen-spatial") { - return runEigenSpatial(input, z, y, x, stride); + return runEigenSpatial(input, n, z, y, x, stride); } else { - return runAtenConv2d(input, z, y, x, stride); + return runAtenConv2d(input, n, z, y, x, stride); } }); } @@ -226,8 +262,8 @@ class Conv2DSelect: public SelectionComp{ unsigned int stride; - static int calculateOutputDimension(int inputDimention, int filterDimention, int stride) { - return (inputDimention - filterDimention) / stride + 1; + static int calculateOutputDimension(int inputDimension, int filterDimension, int stride) { + return (inputDimension - filterDimension) / stride + 1; } }; diff --git a/src/objectModel/headers/BuiltinPDBObjects.h b/src/objectModel/headers/BuiltinPDBObjects.h index cd9077d2..a6195115 100644 --- a/src/objectModel/headers/BuiltinPDBObjects.h +++ b/src/objectModel/headers/BuiltinPDBObjects.h @@ -1,140 +1,138 @@ // Auto-generated by code in SConstruct -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/SetScan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StoragePagePinned.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/KMeansDoubleVector.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveDatabase.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/AggregationJobStage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageRemoveDatabase.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ComputePlan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageTestSetCopy.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ExecuteQuery.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ResourceInfo.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/NodeInfo.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageCollectStatsResponse.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSharedPage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatCreateDatabaseRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetSetRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddObjectInLoop.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PDBObjectPrototype.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ZB_Company.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Array.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/AggregationMap.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DepartmentEmployeeAges.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageCleanup.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Employee.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BroadcastJoinBuildHTJobStage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Avg.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetDatabaseRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetTypeResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddData.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/OptimizedDepartmentEmployees.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ExecuteComputation.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BuiltinPartialResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DepartmentEmployees.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/HashPartitionedJoinBuildHTJobStage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageRemoveTempSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddSharedMapping.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageCleanup.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveTempSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/JoinPairArray.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageNoMorePage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddTempSetResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageClearSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StoragePinPage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StringIntPair.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatPrintCatalogRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/JoinMap.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/GenericBlock.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DoubleSumResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageCollectStats.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CloseConnection.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueryPermitResponse.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSharedLibraryByNameRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DispatcherAddData.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddTempSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DeleteSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/OptimizedSupervisor.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSyncRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TopKQueue.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/OptimizedEmployee.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BaseQuery.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/SumResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/RequestResources.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatDeleteDatabaseRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/WriteUserSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageUnpinPage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/AvgResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Count.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DepartmentTotal.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatRegisterType.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DoubleVectorResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageGetData.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Holder.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StoragePinBytes.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetDatabaseResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ScanDoubleVectorSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageExportSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BackendTestSetCopy.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BackendTestSetScan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageGetSetPages.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddTempSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddSharedPage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageGetDataResponse.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/SetIdentifier.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatTypeNameSearchResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageExportSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Ack.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TupleSetExecuteQuery.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/MyEmployee.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddDatabase.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DoubleVector.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageBytesPinned.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatPrintCatalogResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageTestSetScan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageGetStats.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PairArray.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueryDone.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSyncResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ShutDown.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DispatcherRegisterPartitionPolicy.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageRemoveHashSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/SimpleRequestResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatDeleteSetRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueriesAndPlan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatalogUserTypeMetadata.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveHashSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetType.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddObject.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSetWithPartition.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageClearSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetSetResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/LambdaIdentifier.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DoneWithResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ListOfNodes.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ProcessorFactory.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSetObjectTypeRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddType.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageRemoveUserSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/KeepGoing.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PDBVector.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PDBMap.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/NodeDispatcherData.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Set.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ScanUserSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueryOutput.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PlaceOfQueryPlanner.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BackendExecuteSelection.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Supervisor.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Nothing.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSharedLibraryResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TensorBlockMeta.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSharedMapping.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddDatabase.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TupleSetJobStage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueryPermit.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/GetListOfNodes.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatCreateSetRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TensorBlockIdentifier.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Ack.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/AggregationJobStage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/AggregationMap.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Array.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Avg.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/AvgResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BackendExecuteSelection.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BackendTestSetCopy.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BackendTestSetScan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BaseQuery.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BroadcastJoinBuildHTJobStage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BuiltinPartialResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatalogUserTypeMetadata.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatCreateDatabaseRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatCreateSetRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatDeleteDatabaseRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatDeleteSetRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetDatabaseRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetDatabaseResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetSetRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetSetResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetType.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetTypeResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatPrintCatalogRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatPrintCatalogResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatRegisterType.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSetObjectTypeRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSharedLibraryByNameRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSharedLibraryResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSyncRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSyncResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatTypeNameSearchResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CloseConnection.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ComputePlan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Count.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DeleteSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DepartmentEmployeeAges.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DepartmentEmployees.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DepartmentTotal.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DispatcherAddData.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DispatcherRegisterPartitionPolicy.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddDatabase.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSetWithPartition.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSharedPage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddTempSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageCleanup.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageClearSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageExportSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveDatabase.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveHashSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveTempSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DoneWithResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DoubleSumResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DoubleVector.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DoubleVectorResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Employee.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ExecuteComputation.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ExecuteQuery.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/GenericBlock.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/GetListOfNodes.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/HashPartitionedJoinBuildHTJobStage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Holder.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/JoinMap.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/JoinPairArray.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/KeepGoing.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/KMeansDoubleVector.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/LambdaIdentifier.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ListOfNodes.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/MyEmployee.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/NodeDispatcherData.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/NodeInfo.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Nothing.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/OptimizedDepartmentEmployees.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/OptimizedEmployee.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/OptimizedSupervisor.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PairArray.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PDBMap.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PDBObjectPrototype.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PDBVector.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PlaceOfQueryPlanner.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ProcessorFactory.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueriesAndPlan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueryDone.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueryOutput.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueryPermit.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueryPermitResponse.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/RequestResources.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ResourceInfo.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ScanDoubleVectorSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ScanUserSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Set.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/SetIdentifier.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/SetScan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ShutDown.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/SimpleRequestResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddData.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddDatabase.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddObject.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddObjectInLoop.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddSharedPage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddTempSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddTempSetResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddType.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageBytesPinned.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageCleanup.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageClearSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageCollectStats.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageCollectStatsResponse.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageExportSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageGetData.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageGetDataResponse.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageGetSetPages.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageGetStats.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageNoMorePage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StoragePagePinned.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StoragePinBytes.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StoragePinPage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageRemoveDatabase.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageRemoveHashSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageRemoveTempSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageRemoveUserSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageTestSetCopy.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageTestSetScan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageUnpinPage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StringIntPair.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/SumResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Supervisor.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TensorBlockIdentifier.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TensorBlockMeta.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TopKQueue.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TupleSetExecuteQuery.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TupleSetJobStage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/WriteUserSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ZB_Company.h" diff --git a/src/queries/headers/QueryClient.h b/src/queries/headers/QueryClient.h index 09ae978c..f1a637c6 100644 --- a/src/queries/headers/QueryClient.h +++ b/src/queries/headers/QueryClient.h @@ -160,30 +160,64 @@ if (typeName != getTypeName ()) { bool executeComputations(std::string& errMsg, std::string jobName = "", bool preCompile = false) { // this is the request + std::cout << "------------------------------------------------------------------------------------------------inside execute computations----------------------------" << std::endl; const UseTemporaryAllocationBlock myBlock{256 * 1024 * 1024}; + auto begin = std::chrono::high_resolution_clock::now(); QueryGraphAnalyzer queryAnalyzer(this->queryGraph); + auto end = std::chrono::high_resolution_clock::now(); + std::cout << "Query Analyzer Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; + + begin = std::chrono::high_resolution_clock::now(); std::string tcapString = queryAnalyzer.parseTCAPString(); + end = std::chrono::high_resolution_clock::now(); + std::cout << "Query Parser Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; + + + begin = std::chrono::high_resolution_clock::now(); std::vector> computations; queryAnalyzer.parseComputations(computations); + end = std::chrono::high_resolution_clock::now(); + std::cout << "parseComputations Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; + + begin = std::chrono::high_resolution_clock::now(); Handle>> computationsToSend = makeObject>>(); for (int i = 0; i < computations.size(); i++) { computationsToSend->push_back(computations[i]); } + end = std::chrono::high_resolution_clock::now(); + std::cout << "computationsToSend Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; if (jobName == "") { jobName = clientName + "-" + std::to_string(queryId); queryId ++; } + std::cout << "-----------------------------------------jobName is --------------------------------------------------------------------" << jobName << std::endl; std::cout << "jobName is " << jobName << std::endl; + + begin = std::chrono::high_resolution_clock::now(); Handle executeComputation = makeObject( jobName, tcapString, preCompile); + end = std::chrono::high_resolution_clock::now(); + std::cout << "executeComputation Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; // this call asks the database to execute the query, and then it inserts the result set name // within each of the results, as well as the database connection information // this is for query scheduling stuff if (useScheduler == true) { + cout << "-----------------------------------------Inside useScheduler-----------------------------------------" << std::endl; + begin = std::chrono::high_resolution_clock::now(); return simpleDoubleRequest>, SimpleRequestResult, @@ -212,7 +246,10 @@ if (typeName != getTypeName ()) { executeComputation, computationsToSend); - + end = std::chrono::high_resolution_clock::now(); + std::cout << "useScheduler Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; } else { errMsg = "This query must be sent to QuerySchedulerServer, but it seems " diff --git a/src/queryExecution/source/PipelineStage.cc b/src/queryExecution/source/PipelineStage.cc index 0700504f..d93fc8d1 100644 --- a/src/queryExecution/source/PipelineStage.cc +++ b/src/queryExecution/source/PipelineStage.cc @@ -907,9 +907,14 @@ void PipelineStage::executePipelineWork(int i, << "-" << this->jobStage->getTargetComputationSpecifier() << std::chrono::duration_cast>(end - begin).count() << " secs." << std::endl; + auto start = std::chrono::high_resolution_clock::now(); std::cout << i<<": Running Pipeline\n"; curPipeline->id = i; curPipeline->run(); + end = std::chrono::high_resolution_clock::now(); + std::cout << "RunPipeline Duration : " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; curPipeline = nullptr; newPlan->nullifyPlanPointer(); getAllocator().setPolicy(AllocatorPolicy::defaultAllocator); diff --git a/src/tests/source/Conv2dProjTest.cc b/src/tests/source/Conv2dProjTest.cc index cdacb0ff..12232023 100644 --- a/src/tests/source/Conv2dProjTest.cc +++ b/src/tests/source/Conv2dProjTest.cc @@ -11,12 +11,14 @@ using namespace std; -void load_rnd_img(int x, int y, int z, int size, pdb::PDBClient &pdbClient, +void load_rnd_img(int x, int y, int z, int n, int size, pdb::PDBClient &pdbClient, pdb::String dbName, pdb::String setName) { std::string errMsg; - pdb::makeObjectAllocatorBlock(64 * 1024 * 1024, true); - + pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); + + std::cout << "----------------inside load rnd img---------------: " << x << " " << y << " " << z << " " << n << " " << size << std::endl; pdb::Handle> dimensions = pdb::makeObject>(); + dimensions->push_back(n); dimensions->push_back(z); dimensions->push_back(y); dimensions->push_back(x); @@ -37,16 +39,12 @@ void load_rnd_img(int x, int y, int z, int size, pdb::PDBClient &pdbClient, for (int i = 0; i < size; i++) { try { - pdb::Handle image = pdb::makeObject(3, dimensions); - for (int c = 0; c < z; c++) { - for (int h = 0; h < y; h++) { - for (int w = 0; w < x; w++) { - double data = (bool)gen() ? distn(e2) : distp(e2); - (*(image->rawData))[c * y * x + h * x + w] = data; - } - } - } - images->push_back(image); + pdb::Handle image = pdb::makeObject(4, dimensions); + for (unsigned int i = 0; i < n*z*y*x; i++) { + float data = (bool)gen() ? distn(e2) : distp(e2); + (*(image->rawData))[i] = data; + } + images->push_back(image); } catch (pdb::NotEnoughSpace &e) { if (!pdbClient.sendData( pair(setName, dbName), images, errMsg)) { @@ -54,8 +52,9 @@ void load_rnd_img(int x, int y, int z, int size, pdb::PDBClient &pdbClient, exit(1); } i--; - pdb::makeObjectAllocatorBlock(64 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); dimensions = pdb::makeObject>(); + dimensions->push_back(n); dimensions->push_back(z); dimensions->push_back(y); dimensions->push_back(x); @@ -190,7 +189,7 @@ int main(int argc, char *argv[]) { pdbClient.removeSet(dbName, img_set, errMsg); if (!pdbClient.createSet( - dbName, img_set, errMsg, (size_t)128 * (size_t)1024 * (size_t)1024)) { + dbName, img_set, errMsg, (size_t)512 * (size_t)1024 * (size_t)1024)) { cout << "Not able to create set " + img_set + ": " + errMsg; } else { cout << "Created set " << img_set << ".\n"; @@ -199,21 +198,21 @@ int main(int argc, char *argv[]) { //load data if (addDataOrNot == true) { - load_rnd_img(w, h, c, numImages, pdbClient, dbName, img_set); + load_rnd_img(w, h, c, n, numImages, pdbClient, dbName, img_set); } //create output dataset string feature_out_set = "feature_map"; pdbClient.removeSet(dbName, feature_out_set, errMsg); if (!pdbClient.createSet( - dbName, feature_out_set, errMsg, (size_t)128 * (size_t)1024 * (size_t)1024)) { + dbName, feature_out_set, errMsg, (size_t)1024 * (size_t)1024 * (size_t)1024)) { cout << "Not able to create set " + feature_out_set + ": " + errMsg; } else { cout << "Created set " << feature_out_set << ".\n"; } auto begin1 = std::chrono::high_resolution_clock::now(); - pdb::makeObjectAllocatorBlock(64 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(8 * 1024 * 1024, true); //create scan computation pdb::Handle imageScanner = @@ -249,11 +248,13 @@ int main(int argc, char *argv[]) { auto begin = std::chrono::high_resolution_clock::now(); + + std::cout<< "---------------------------Starting execute computations---------------------------" << std::endl; // run the computation if (!pdbClient.executeComputations(errMsg, "conv2d-proj", myWriteSet)) { cout << "Computation failed. Message was: " << errMsg << "\n"; exit(1); - } + } auto end = std::chrono::high_resolution_clock::now();