diff --git a/src/FF/headers/FFInputLayerJoin.h b/src/FF/headers/FFInputLayerJoin.h index 383d6c2f..7b36a7ae 100644 --- a/src/FF/headers/FFInputLayerJoin.h +++ b/src/FF/headers/FFInputLayerJoin.h @@ -1,93 +1,93 @@ -#ifndef FF_INPUTLAYER_JOIN_H -#define FF_INPUTLAYER_JOIN_H - -#include "FFMatrixBlock.h" -#include "JoinComp.h" -#include "Lambda.h" -#include "LambdaCreationFunctions.h" - -// LA libraries: -#include - -using namespace pdb; - -class FFInputLayerJoin - : public JoinComp { - -public: - ENABLE_DEEP_COPY - - FFInputLayerJoin() {} - - Lambda getSelection(Handle in1, - Handle in2) override { - // return makeLambda( - - // in1, in2, [](Handle &in1, Handle &in2) - // { - // return in1->getBlockColIndex() == in2->getBlockRowIndex(); - // }); - return makeLambdaFromMethod(in1, getBlockColIndex) == - makeLambdaFromMethod(in2, getBlockRowIndex); - } - - Lambda> - getProjection(Handle in1, Handle in2) override { - return makeLambda( - in1, in2, [](Handle &in1, Handle &in2) { - if (FFMatrixBlock::librayCode == EIGEN_CODE) { - // get the sizes - uint32_t I = in1->getRowNums(); - uint32_t J = in2->getColNums(); - uint32_t K = in1->getColNums(); - - // make an output - pdb::Handle resultFFMatrixBlock = - pdb::makeObject( - in1->getBlockRowIndex(), in2->getBlockColIndex(), I, J, - in1->getTotalRowNums(), in2->getTotalColNums(), false); - - // get the ptrs - double *outData = resultFFMatrixBlock->getValue().rawData->c_ptr(); - double *in1Data = in1->getValue().rawData->c_ptr(); - double *in2Data = in2->getValue().rawData->c_ptr(); - - // do the multiply - Eigen::Map> - currentMatrix1(in1->getValue().rawData->c_ptr(), - in1->getRowNums(), in1->getColNums()); - Eigen::Map> - currentMatrix2(in2->getValue().rawData->c_ptr(), - in2->getRowNums(), in2->getColNums()); - - Eigen::Map> - productMatrix(resultFFMatrixBlock->getValue().rawData->c_ptr(), - I, J); - - productMatrix = currentMatrix1 * currentMatrix2; - - // process the bias if necessary - if (in2->getValue().bias != nullptr) { - auto bias = in2->getValue().bias->c_ptr(); - for (uint32_t r = 0; r < I; r++) { - for (uint32_t c = 0; c < J; c++) { - - // add the bias - outData[r * J + c] += bias[c]; - } - } - } - - return resultFFMatrixBlock; - } else { - std::cerr << "Wrong librayCode!" << std::endl; - exit(1); - } - }); - } -}; - -#endif +#ifndef FF_INPUTLAYER_JOIN_H +#define FF_INPUTLAYER_JOIN_H + +#include "FFMatrixBlock.h" +#include "JoinComp.h" +#include "Lambda.h" +#include "LambdaCreationFunctions.h" + +// LA libraries: +#include + +using namespace pdb; + +class FFInputLayerJoin + : public JoinComp { + +public: + ENABLE_DEEP_COPY + + FFInputLayerJoin() {} + + Lambda getSelection(Handle in1, + Handle in2) override { + // return makeLambda( + + // in1, in2, [](Handle &in1, Handle &in2) + // { + // return in1->getBlockColIndex() == in2->getBlockRowIndex(); + // }); + return makeLambdaFromMethod(in1, getBlockColIndex) == + makeLambdaFromMethod(in2, getBlockRowIndex); + } + + Lambda> + getProjection(Handle in1, Handle in2) override { + return makeLambda( + in1, in2, [](Handle &in1, Handle &in2) { + if (FFMatrixBlock::librayCode == EIGEN_CODE) { + // get the sizes + uint32_t I = in1->getRowNums(); + uint32_t J = in2->getColNums(); + uint32_t K = in1->getColNums(); + + // make an output + pdb::Handle resultFFMatrixBlock = + pdb::makeObject( + in1->getBlockRowIndex(), in2->getBlockColIndex(), I, J, + in1->getTotalRowNums(), in2->getTotalColNums(), false); + + // get the ptrs + float *outData = resultFFMatrixBlock->getValue().rawData->c_ptr(); + float *in1Data = in1->getValue().rawData->c_ptr(); + float *in2Data = in2->getValue().rawData->c_ptr(); + + // do the multiply + Eigen::Map> + currentMatrix1(in1->getValue().rawData->c_ptr(), + in1->getRowNums(), in1->getColNums()); + Eigen::Map> + currentMatrix2(in2->getValue().rawData->c_ptr(), + in2->getRowNums(), in2->getColNums()); + + Eigen::Map> + productMatrix(resultFFMatrixBlock->getValue().rawData->c_ptr(), + I, J); + + productMatrix = currentMatrix1 * currentMatrix2; + + // process the bias if necessary + if (in2->getValue().bias != nullptr) { + auto bias = in2->getValue().bias->c_ptr(); + for (uint32_t r = 0; r < I; r++) { + for (uint32_t c = 0; c < J; c++) { + + // add the bias + outData[r * J + c] += bias[c]; + } + } + } + + return resultFFMatrixBlock; + } else { + std::cerr << "Wrong librayCode!" << std::endl; + exit(1); + } + }); + } +}; + +#endif diff --git a/src/FF/headers/FFMatrixBlock.h b/src/FF/headers/FFMatrixBlock.h index 3f959ade..b7bf599f 100644 --- a/src/FF/headers/FFMatrixBlock.h +++ b/src/FF/headers/FFMatrixBlock.h @@ -1,152 +1,152 @@ -#ifndef FF_MATRIXBLOCK_H -#define FF_MATRIXBLOCK_H - -#ifndef EIGEN_CODE -#define EIGEN_CODE 0 -#endif - -#include "FFMatrixData.h" -#include "FFMatrixMeta.h" -#include "Handle.h" -#include "PDBString.h" -#include "PDBVector.h" -#include "StringIntPair.h" - -// LA libraries: -#include - -class FFMatrixBlock : public pdb::Object { -public: - FFMatrixData data; - pdb::Handle meta; - bool partitionByCol = true; - const static int librayCode = EIGEN_CODE; - - ENABLE_DEEP_COPY - - ~FFMatrixBlock() {} - - FFMatrixBlock() {} - - FFMatrixBlock(int blockRowIndexIn, int blockColIndexIn, int rowNumsIn, - int colNumsIn, bool partitionByCol = true) - : data(rowNumsIn, colNumsIn) { - meta = pdb::makeObject(blockRowIndexIn, blockColIndexIn); - this->partitionByCol = partitionByCol; - } - - FFMatrixBlock(int blockRowIndexIn, int blockColIndexIn, int rowNumsIn, - int colNumsIn, int totalRows, int totalCols, bool partitionByCol = true) - : data(rowNumsIn, colNumsIn) { - meta = pdb::makeObject(blockRowIndexIn, blockColIndexIn, totalRows, totalCols); - this->partitionByCol = partitionByCol; - } - - FFMatrixBlock(int blockRowIndexIn, int blockColIndexIn, int rowNumsIn, - int colNumsIn, pdb::Handle> rawDataIn, bool partitionByCol = true) - : data(rowNumsIn, colNumsIn, rawDataIn) { - meta = pdb::makeObject(blockRowIndexIn, blockColIndexIn); - this->partitionByCol = partitionByCol; - } - - FFMatrixBlock(int blockRowIndexIn, int blockColIndexIn, int rowNumsIn, - int colNumsIn, int totalRows, int totalCols, - pdb::Handle> rawDataIn, bool partitionByCol = true) - : data(rowNumsIn, colNumsIn, rawDataIn) { - meta = pdb::makeObject(blockRowIndexIn, blockColIndexIn, totalRows, totalCols); - this->partitionByCol = partitionByCol; - } - - int getRowNums() { return data.rowNums; } - - int rowIndexStart() { - return getBlockRowIndex() * getRowNums(); - } - - int rowIndexEnd() { - return getBlockRowIndex() * getRowNums() + getRowNums(); - } - - bool isLastColBlock() { - int numYBlocks = ceil(getTotalColNums() / (double)getColNums()); - return getBlockColIndex() == numYBlocks - 1; - } - - bool isLastRowBlock() { - int numXBlocks = ceil(getTotalRowNums() / (double)getRowNums()); - return getBlockRowIndex() == numXBlocks - 1; - } - - int getColNums() { return data.colNums; } - - pdb::Handle> &getRawDataHandle() { return data.rawData; } - - pdb::Handle &getKey() { return meta; } - - FFMatrixMeta &getKeyRef() { return *meta; } - - FFMatrixData &getValue() { return data; } - - int getBlockRowIndex() { return meta->blockRowIndex; } - - int getBlockColIndex() { return meta->blockColIndex; } - - int getTotalRowNums() { return meta->totalRows; } - - int getTotalColNums() { return meta->totalCols; } - - int getNumRowBlocks() { - return ceil(getTotalRowNums() / (double)getRowNums()); - } - - int getNumColBlocks() { - return ceil(getTotalColNums() / (double)getColNums()); - } - - // This is needed for row-wise computation - pdb::Handle getRowKey() { - pdb::Handle rowMeta = pdb::makeObject(meta->blockRowIndex, 0, meta->totalRows, 1); - return rowMeta; - } - - // This is needed for row-wise computation - FFMatrixData getRowSumValue() { - FFMatrixData sumRowData; - sumRowData.rowNums = data.rowNums; - sumRowData.colNums = 1; - int bufferLength = sumRowData.rowNums * sumRowData.colNums; - sumRowData.rawData = pdb::makeObject>(bufferLength, bufferLength); - - Eigen::Map> - currentMatrix((data.rawData)->c_ptr(), data.rowNums, data.colNums); - - Eigen::Map> - rowSumMatrix((sumRowData.rawData)->c_ptr(), sumRowData.rowNums, sumRowData.colNums); - - rowSumMatrix = currentMatrix.rowwise().sum(); - - // std::cout <<"getRowSumValue Matrix :"<< std::endl; - // this->print(); - // std::cout <<"Row wise Sum Matrix :"<< std::endl; - // sumRowData.print(); - return sumRowData; - } - - void print() { - std::cout << "BlockRowIndex:" << getBlockRowIndex() << std::endl; - std::cout << "BlockColumnIndex:" << getBlockColIndex() << std::endl; - std::cout << "TotalRowNums:" << getTotalRowNums() << std::endl; - std::cout << "TotalColNums:" << getTotalColNums() << std::endl; - } - - size_t hash() const override{ - if (partitionByCol) - return pdb::Hasher::hash(meta->blockColIndex); - else - return pdb::Hasher::hash(meta->blockRowIndex); - } - - -}; - -#endif +#ifndef FF_MATRIXBLOCK_H +#define FF_MATRIXBLOCK_H + +#ifndef EIGEN_CODE +#define EIGEN_CODE 0 +#endif + +#include "FFMatrixData.h" +#include "FFMatrixMeta.h" +#include "Handle.h" +#include "PDBString.h" +#include "PDBVector.h" +#include "StringIntPair.h" + +// LA libraries: +#include + +class FFMatrixBlock : public pdb::Object { +public: + FFMatrixData data; + pdb::Handle meta; + bool partitionByCol = true; + const static int librayCode = EIGEN_CODE; + + ENABLE_DEEP_COPY + + ~FFMatrixBlock() {} + + FFMatrixBlock() {} + + FFMatrixBlock(int blockRowIndexIn, int blockColIndexIn, int rowNumsIn, + int colNumsIn, bool partitionByCol = true) + : data(rowNumsIn, colNumsIn) { + meta = pdb::makeObject(blockRowIndexIn, blockColIndexIn); + this->partitionByCol = partitionByCol; + } + + FFMatrixBlock(int blockRowIndexIn, int blockColIndexIn, int rowNumsIn, + int colNumsIn, int totalRows, int totalCols, bool partitionByCol = true) + : data(rowNumsIn, colNumsIn) { + meta = pdb::makeObject(blockRowIndexIn, blockColIndexIn, totalRows, totalCols); + this->partitionByCol = partitionByCol; + } + + FFMatrixBlock(int blockRowIndexIn, int blockColIndexIn, int rowNumsIn, + int colNumsIn, pdb::Handle> rawDataIn, bool partitionByCol = true) + : data(rowNumsIn, colNumsIn, rawDataIn) { + meta = pdb::makeObject(blockRowIndexIn, blockColIndexIn); + this->partitionByCol = partitionByCol; + } + + FFMatrixBlock(int blockRowIndexIn, int blockColIndexIn, int rowNumsIn, + int colNumsIn, int totalRows, int totalCols, + pdb::Handle> rawDataIn, bool partitionByCol = true) + : data(rowNumsIn, colNumsIn, rawDataIn) { + meta = pdb::makeObject(blockRowIndexIn, blockColIndexIn, totalRows, totalCols); + this->partitionByCol = partitionByCol; + } + + int getRowNums() { return data.rowNums; } + + int rowIndexStart() { + return getBlockRowIndex() * getRowNums(); + } + + int rowIndexEnd() { + return getBlockRowIndex() * getRowNums() + getRowNums(); + } + + bool isLastColBlock() { + int numYBlocks = ceil(getTotalColNums() / (double)getColNums()); + return getBlockColIndex() == numYBlocks - 1; + } + + bool isLastRowBlock() { + int numXBlocks = ceil(getTotalRowNums() / (double)getRowNums()); + return getBlockRowIndex() == numXBlocks - 1; + } + + int getColNums() { return data.colNums; } + + pdb::Handle> &getRawDataHandle() { return data.rawData; } + + pdb::Handle &getKey() { return meta; } + + FFMatrixMeta &getKeyRef() { return *meta; } + + FFMatrixData &getValue() { return data; } + + int getBlockRowIndex() { return meta->blockRowIndex; } + + int getBlockColIndex() { return meta->blockColIndex; } + + int getTotalRowNums() { return meta->totalRows; } + + int getTotalColNums() { return meta->totalCols; } + + int getNumRowBlocks() { + return ceil(getTotalRowNums() / (double)getRowNums()); + } + + int getNumColBlocks() { + return ceil(getTotalColNums() / (double)getColNums()); + } + + // This is needed for row-wise computation + pdb::Handle getRowKey() { + pdb::Handle rowMeta = pdb::makeObject(meta->blockRowIndex, 0, meta->totalRows, 1); + return rowMeta; + } + + // This is needed for row-wise computation + FFMatrixData getRowSumValue() { + FFMatrixData sumRowData; + sumRowData.rowNums = data.rowNums; + sumRowData.colNums = 1; + int bufferLength = sumRowData.rowNums * sumRowData.colNums; + sumRowData.rawData = pdb::makeObject>(bufferLength, bufferLength); + + Eigen::Map> + currentMatrix((data.rawData)->c_ptr(), data.rowNums, data.colNums); + + Eigen::Map> + rowSumMatrix((sumRowData.rawData)->c_ptr(), sumRowData.rowNums, sumRowData.colNums); + + rowSumMatrix = currentMatrix.rowwise().sum(); + + // std::cout <<"getRowSumValue Matrix :"<< std::endl; + // this->print(); + // std::cout <<"Row wise Sum Matrix :"<< std::endl; + // sumRowData.print(); + return sumRowData; + } + + void print() { + std::cout << "BlockRowIndex:" << getBlockRowIndex() << std::endl; + std::cout << "BlockColumnIndex:" << getBlockColIndex() << std::endl; + std::cout << "TotalRowNums:" << getTotalRowNums() << std::endl; + std::cout << "TotalColNums:" << getTotalColNums() << std::endl; + } + + size_t hash() const override{ + if (partitionByCol) + return pdb::Hasher::hash(meta->blockColIndex); + else + return pdb::Hasher::hash(meta->blockRowIndex); + } + + +}; + +#endif diff --git a/src/FF/headers/FFMatrixData.h b/src/FF/headers/FFMatrixData.h index 6116d8cb..ee6fa93c 100644 --- a/src/FF/headers/FFMatrixData.h +++ b/src/FF/headers/FFMatrixData.h @@ -1,60 +1,60 @@ -#ifndef FF_MATRIXDATA_H -#define FF_MATRIXDATA_H - -#include "Handle.h" -#include "Object.h" -#include "PDBString.h" -#include "PDBVector.h" -#include "StringIntPair.h" - - -class FFMatrixData : public pdb::Object { -public: - int rowNums = 0; - int colNums = 0; - - ENABLE_DEEP_COPY - - ~FFMatrixData() {} - - FFMatrixData(int rowNumsIn, int colNumsIn) - : rowNums(rowNumsIn), colNums(colNumsIn) { - rawData = pdb::makeObject>(rowNumsIn * colNumsIn, - rowNumsIn * colNumsIn); - } - - FFMatrixData(int rowNumsIn, int colNumsIn, - pdb::Handle> rawDataIn) - : rowNums(rowNumsIn), colNums(colNumsIn), rawData(rawDataIn) { - rawData = pdb::makeObject>(rowNumsIn * colNumsIn, - rowNumsIn * colNumsIn); - } - - FFMatrixData() {} - - pdb::Handle> rawData; - pdb::Handle> bias; - - FFMatrixData &operator+(FFMatrixData &other) { - double *myData, *otherData; - - myData = rawData->c_ptr(); - otherData = other.rawData->c_ptr(); - - for (int i = 0; i < rowNums * colNums; i++) { - (myData)[i] += (otherData)[i]; - } - - if (bias != nullptr && other.bias != nullptr) { - myData = bias->c_ptr(); - otherData = other.bias->c_ptr(); - for (int i = 0; i < other.bias->size(); i++) { - (myData)[i] += (otherData)[i]; - } - } - - return *this; - } -}; - -#endif +#ifndef FF_MATRIXDATA_H +#define FF_MATRIXDATA_H + +#include "Handle.h" +#include "Object.h" +#include "PDBString.h" +#include "PDBVector.h" +#include "StringIntPair.h" + + +class FFMatrixData : public pdb::Object { +public: + int rowNums = 0; + int colNums = 0; + + ENABLE_DEEP_COPY + + ~FFMatrixData() {} + + FFMatrixData(int rowNumsIn, int colNumsIn) + : rowNums(rowNumsIn), colNums(colNumsIn) { + rawData = pdb::makeObject>(rowNumsIn * colNumsIn, + rowNumsIn * colNumsIn); + } + + FFMatrixData(int rowNumsIn, int colNumsIn, + pdb::Handle> rawDataIn) + : rowNums(rowNumsIn), colNums(colNumsIn), rawData(rawDataIn) { + rawData = pdb::makeObject>(rowNumsIn * colNumsIn, + rowNumsIn * colNumsIn); + } + + FFMatrixData() {} + + pdb::Handle> rawData; + pdb::Handle> bias; + + FFMatrixData &operator+(FFMatrixData &other) { + float *myData, *otherData; + + myData = rawData->c_ptr(); + otherData = other.rawData->c_ptr(); + + for (int i = 0; i < rowNums * colNums; i++) { + (myData)[i] += (otherData)[i]; + } + + if (bias != nullptr && other.bias != nullptr) { + myData = bias->c_ptr(); + otherData = other.bias->c_ptr(); + for (int i = 0; i < other.bias->size(); i++) { + (myData)[i] += (otherData)[i]; + } + } + + return *this; + } +}; + +#endif diff --git a/src/FF/headers/FFMatrixMultiSel.h b/src/FF/headers/FFMatrixMultiSel.h index 69e9f374..add8cae1 100644 --- a/src/FF/headers/FFMatrixMultiSel.h +++ b/src/FF/headers/FFMatrixMultiSel.h @@ -1,63 +1,63 @@ -#ifndef FF_MATRIX_MULTISEL_H -#define FF_MATRIX_MULTISEL_H - -#include -#include - -#include "MultiSelectionComp.h" - -#include "Lambda.h" -#include "LambdaCreationFunctions.h" - -#include "PDBString.h" -#include "PDBVector.h" - -#include "FFMatrixBlock.h" -#include "InferenceResult.h" - -using namespace pdb; - -class FFMatrixMultiSel - : public MultiSelectionComp { - -public: - ENABLE_DEEP_COPY - - FFMatrixMultiSel() {} - - Lambda getSelection(Handle checkMe) override { - return makeLambda( - checkMe, [](Handle &checkMe) { return true; }); - } - - Lambda>> - getProjection(Handle checkMe) override { - return makeLambda(checkMe, [this](Handle &checkMe) { - Vector> result; - - uint32_t I = checkMe->getRowNums(); - // This is the inference output with the probabilities of each label. We have only 2 labels. - uint32_t J = checkMe->getColNums(); - - // assert(J == 2); - - double *checkMeData = checkMe->getValue().rawData->c_ptr(); - - for (int i = 0; i < I; i++) { - int index = checkMe->getBlockRowIndex() * I + i; - Handle inference = makeObject(index, checkMe->getBlockRowIndex()); - Vector &inference_results = inference->getInference(); - for (int j = 0; j < 2; j++) { - inference_results.push_back(checkMeData[i * J + j]); - } - inference->dump(); - result.push_back(inference); - } - - return result; - }); - } -}; - - -#endif +#ifndef FF_MATRIX_MULTISEL_H +#define FF_MATRIX_MULTISEL_H + +#include +#include + +#include "MultiSelectionComp.h" + +#include "Lambda.h" +#include "LambdaCreationFunctions.h" + +#include "PDBString.h" +#include "PDBVector.h" + +#include "FFMatrixBlock.h" +#include "InferenceResult.h" + +using namespace pdb; + +class FFMatrixMultiSel + : public MultiSelectionComp { + +public: + ENABLE_DEEP_COPY + + FFMatrixMultiSel() {} + + Lambda getSelection(Handle checkMe) override { + return makeLambda( + checkMe, [](Handle &checkMe) { return true; }); + } + + Lambda>> + getProjection(Handle checkMe) override { + return makeLambda(checkMe, [this](Handle &checkMe) { + Vector> result; + + uint32_t I = checkMe->getRowNums(); + // This is the inference output with the probabilities of each label. We have only 2 labels. + uint32_t J = checkMe->getColNums(); + + // assert(J == 2); + + float *checkMeData = checkMe->getValue().rawData->c_ptr(); + + for (int i = 0; i < I; i++) { + int index = checkMe->getBlockRowIndex() * I + i; + Handle inference = makeObject(index, checkMe->getBlockRowIndex()); + Vector &inference_results = inference->getInference(); + for (int j = 0; j < 2; j++) { + inference_results.push_back(checkMeData[i * J + j]); + } + inference->dump(); + result.push_back(inference); + } + + return result; + }); + } +}; + + +#endif diff --git a/src/FF/headers/FFMatrixUtil.h b/src/FF/headers/FFMatrixUtil.h index 390c3b7b..0abd6f51 100644 --- a/src/FF/headers/FFMatrixUtil.h +++ b/src/FF/headers/FFMatrixUtil.h @@ -1,40 +1,40 @@ -#include -#include -#include -#include -#include -#include - -namespace pdb { -class PDBClient; -class CatalogClient; -class String; -} // namespace pdb - -namespace ff { -void load_matrix_data(pdb::PDBClient &pdbClient, std::string path, - pdb::String dbName, pdb::String setName, int blockX, - int blockY, bool dont_pad_x, bool dont_pad_y, - std::string &errMsg, int size = 128, bool partitionByCol = true); - -void load_matrix_data(pdb::PDBClient &pdbClient, std::string path, - pdb::String dbName, pdb::String setName, int pad_x, - int pad_y, std::string &errMsg); - -void loadMatrix(pdb::PDBClient &pdbClient, pdb::String dbName, - pdb::String setName, int totalX, int totalY, int blockX, - int blockY, bool dont_pad_x, bool dont_pad_y, - std::string &errMsg, int size = 128, - bool partitionByCol = true); - -void load_matrix_from_file(std::string path, - std::vector> &matrix); - -void print_stats(pdb::PDBClient &pdbClient, std::string dbName, - std::string setName); - -void print(pdb::PDBClient &pdbClient, std::string dbName, std::string setName); - -bool is_empty_set(pdb::PDBClient &pdbClient, pdb::CatalogClient &catalogClient, - std::string dbName, std::string setName); -} // namespace ff +#include +#include +#include +#include +#include +#include + +namespace pdb { +class PDBClient; +class CatalogClient; +class String; +} // namespace pdb + +namespace ff { +void load_matrix_data(pdb::PDBClient &pdbClient, std::string path, + pdb::String dbName, pdb::String setName, int blockX, + int blockY, bool dont_pad_x, bool dont_pad_y, + std::string &errMsg, int size = 128, bool partitionByCol = true); + +void load_matrix_data(pdb::PDBClient &pdbClient, std::string path, + pdb::String dbName, pdb::String setName, int pad_x, + int pad_y, std::string &errMsg); + +void loadMatrix(pdb::PDBClient &pdbClient, pdb::String dbName, + pdb::String setName, int totalX, int totalY, int blockX, + int blockY, bool dont_pad_x, bool dont_pad_y, + std::string &errMsg, int size = 128, + bool partitionByCol = true); + +void load_matrix_from_file(std::string path, + std::vector> &matrix); + +void print_stats(pdb::PDBClient &pdbClient, std::string dbName, + std::string setName); + +void print(pdb::PDBClient &pdbClient, std::string dbName, std::string setName); + +bool is_empty_set(pdb::PDBClient &pdbClient, pdb::CatalogClient &catalogClient, + std::string dbName, std::string setName); +} // namespace ff diff --git a/src/FF/headers/FFOutputLayer.h b/src/FF/headers/FFOutputLayer.h index 000c18cd..d1a117d5 100644 --- a/src/FF/headers/FFOutputLayer.h +++ b/src/FF/headers/FFOutputLayer.h @@ -1,67 +1,67 @@ -#ifndef FF_OUTPUTLAYER_H -#define FF_OUTPUTLAYER_H - -#include "FFMatrixBlock.h" -#include "JoinComp.h" -#include "Lambda.h" -#include "LambdaCreationFunctions.h" - -using namespace pdb; - -class FFOutputLayer - : public JoinComp { - -public: - ENABLE_DEEP_COPY - - FFOutputLayer() {} - - Lambda getSelection(Handle in1, - Handle in2) override { - return makeLambdaFromMethod(in1, getBlockRowIndex) == - makeLambdaFromMethod(in2, getBlockRowIndex); - } - - Lambda> - getProjection(Handle in1, Handle in2) override { - return makeLambda( - in1, in2, [](Handle &in1, Handle &in2) { - if (FFMatrixBlock::librayCode == EIGEN_CODE) { - // get the sizes - uint32_t I = in1->getRowNums(); - uint32_t J = in1->getColNums(); - - if (in1->getRowNums() != in2->getRowNums() || in2->getColNums() != 1) { - cout << "[FFOutputLayer] Cannot divide! Dimensions mismatch!" - << endl; - exit(-1); - } - - // make an output - pdb::Handle resultFFMatrixBlock = - pdb::makeObject( - in1->getBlockRowIndex(), in1->getBlockColIndex(), I, J, - in1->getTotalRowNums(), in1->getTotalColNums(), false); - - // get the ptrs - double *outData = resultFFMatrixBlock->getValue().rawData->c_ptr(); - double *in1Data = in1->getValue().rawData->c_ptr(); - double *in2Data = in2->getValue().rawData->c_ptr(); - - for (int i = 0; i < I; i++) { - for (int j = 0; j < J; j++) { - int pos = i * I + j; - outData[pos] = in1Data[pos] / in2Data[i]; - } - } - - return resultFFMatrixBlock; - } else { - std::cerr << "Wrong librayCode!" << std::endl; - exit(1); - } - }); - } -}; - +#ifndef FF_OUTPUTLAYER_H +#define FF_OUTPUTLAYER_H + +#include "FFMatrixBlock.h" +#include "JoinComp.h" +#include "Lambda.h" +#include "LambdaCreationFunctions.h" + +using namespace pdb; + +class FFOutputLayer + : public JoinComp { + +public: + ENABLE_DEEP_COPY + + FFOutputLayer() {} + + Lambda getSelection(Handle in1, + Handle in2) override { + return makeLambdaFromMethod(in1, getBlockRowIndex) == + makeLambdaFromMethod(in2, getBlockRowIndex); + } + + Lambda> + getProjection(Handle in1, Handle in2) override { + return makeLambda( + in1, in2, [](Handle &in1, Handle &in2) { + if (FFMatrixBlock::librayCode == EIGEN_CODE) { + // get the sizes + uint32_t I = in1->getRowNums(); + uint32_t J = in1->getColNums(); + + if (in1->getRowNums() != in2->getRowNums() || in2->getColNums() != 1) { + cout << "[FFOutputLayer] Cannot divide! Dimensions mismatch!" + << endl; + exit(-1); + } + + // make an output + pdb::Handle resultFFMatrixBlock = + pdb::makeObject( + in1->getBlockRowIndex(), in1->getBlockColIndex(), I, J, + in1->getTotalRowNums(), in1->getTotalColNums(), false); + + // get the ptrs + float *outData = resultFFMatrixBlock->getValue().rawData->c_ptr(); + float *in1Data = in1->getValue().rawData->c_ptr(); + float *in2Data = in2->getValue().rawData->c_ptr(); + + for (int i = 0; i < I; i++) { + for (int j = 0; j < J; j++) { + int pos = i * I + j; + outData[pos] = in1Data[pos] / in2Data[i]; + } + } + + return resultFFMatrixBlock; + } else { + std::cerr << "Wrong librayCode!" << std::endl; + exit(1); + } + }); + } +}; + #endif \ No newline at end of file diff --git a/src/FF/headers/FFReluBiasSum.h b/src/FF/headers/FFReluBiasSum.h index 4e3d535a..c35e1bf5 100644 --- a/src/FF/headers/FFReluBiasSum.h +++ b/src/FF/headers/FFReluBiasSum.h @@ -1,93 +1,93 @@ -#ifndef FF_RELU_BIAS_SUM_H -#define FF_RELU_BIAS_SUM_H - -#include "FFMatrixBlock.h" -#include "JoinComp.h" -#include "Lambda.h" -#include "LambdaCreationFunctions.h" - -#include -#include - -using namespace pdb; - -class FFReluBiasSum - : public JoinComp { - -public: - ENABLE_DEEP_COPY - - FFReluBiasSum() : dropout_rate(0){}; - - FFReluBiasSum(double dropout_rate_in) : dropout_rate(dropout_rate_in){}; - - double dropout_rate; - - Lambda getSelection(Handle in1, - Handle in2) override { - // return makeLambda( - - // in1, in2, [](Handle &in1, Handle &in2) - // { - // return in1->getBlockRowIndex() == in2->getBlockRowIndex(); - // }); - return makeLambdaFromMethod(in1, getBlockRowIndex) == - makeLambdaFromMethod(in2, getBlockRowIndex); - - } - - Lambda> - getProjection(Handle in1, Handle in2) override { - return makeLambda( - in1, in2, [&](Handle &in1, Handle &in2) { - if (FFMatrixBlock::librayCode == EIGEN_CODE) { - // get the sizes - - uint32_t I = in1->getRowNums(); - uint32_t J = in1->getColNums(); - - if (in1->getRowNums() != in2->getRowNums() || - in2->getColNums() != 1) { - std::cout << "[ReluBiasSum] IN1 : " << I << " X " << J - << ", IN2: " << in2->getRowNums() << " X " - << in2->getColNums() << std::endl; - std::cerr << "Block dimemsions mismatch!" << std::endl; - exit(1); - } - - pdb::Handle resultFFMatrixBlock = - pdb::makeObject( - in1->getBlockRowIndex(), in1->getBlockColIndex(), I, J, - in1->getTotalRowNums(), in1->getTotalColNums(), false); - - double *outData = resultFFMatrixBlock->getValue().rawData->c_ptr(); - double *in1Data = in1->getValue().rawData->c_ptr(); - double *in2Data = in2->getValue().rawData->c_ptr(); - - srand(time(NULL)); - for (int32_t i = 0; i < I; i++) { - for (int32_t j = 0; j < J; j++) { - int32_t pos = i * J + j; - outData[pos] = max(0.0, in1Data[pos] + in2Data[i]); - - if (dropout_rate != 0) { - bool zero = (rand() % 100) < (dropout_rate * 100); - if (zero) { - outData[pos] = 0; - } else { - outData[pos] *= 1 / (1 - dropout_rate); - } - } - } - } - - return resultFFMatrixBlock; - } else { - std::cerr << "Wrong librayCode!" << std::endl; - exit(1); - } - }); - } -}; - -#endif +#ifndef FF_RELU_BIAS_SUM_H +#define FF_RELU_BIAS_SUM_H + +#include "FFMatrixBlock.h" +#include "JoinComp.h" +#include "Lambda.h" +#include "LambdaCreationFunctions.h" + +#include +#include + +using namespace pdb; + +class FFReluBiasSum + : public JoinComp { + +public: + ENABLE_DEEP_COPY + + FFReluBiasSum() : dropout_rate(0){}; + + FFReluBiasSum(double dropout_rate_in) : dropout_rate(dropout_rate_in){}; + + double dropout_rate; + + Lambda getSelection(Handle in1, + Handle in2) override { + // return makeLambda( + + // in1, in2, [](Handle &in1, Handle &in2) + // { + // return in1->getBlockRowIndex() == in2->getBlockRowIndex(); + // }); + return makeLambdaFromMethod(in1, getBlockRowIndex) == + makeLambdaFromMethod(in2, getBlockRowIndex); + + } + + Lambda> + getProjection(Handle in1, Handle in2) override { + return makeLambda( + in1, in2, [&](Handle &in1, Handle &in2) { + if (FFMatrixBlock::librayCode == EIGEN_CODE) { + // get the sizes + + uint32_t I = in1->getRowNums(); + uint32_t J = in1->getColNums(); + + if (in1->getRowNums() != in2->getRowNums() || + in2->getColNums() != 1) { + std::cout << "[ReluBiasSum] IN1 : " << I << " X " << J + << ", IN2: " << in2->getRowNums() << " X " + << in2->getColNums() << std::endl; + std::cerr << "Block dimemsions mismatch!" << std::endl; + exit(1); + } + + pdb::Handle resultFFMatrixBlock = + pdb::makeObject( + in1->getBlockRowIndex(), in1->getBlockColIndex(), I, J, + in1->getTotalRowNums(), in1->getTotalColNums(), false); + + float *outData = resultFFMatrixBlock->getValue().rawData->c_ptr(); + float *in1Data = in1->getValue().rawData->c_ptr(); + float *in2Data = in2->getValue().rawData->c_ptr(); + + srand(time(NULL)); + for (int32_t i = 0; i < I; i++) { + for (int32_t j = 0; j < J; j++) { + int32_t pos = i * J + j; + outData[pos] = max(0.0, in1Data[pos] + in2Data[i]); + + if (dropout_rate != 0) { + bool zero = (rand() % 100) < (dropout_rate * 100); + if (zero) { + outData[pos] = 0; + } else { + outData[pos] *= 1 / (1 - dropout_rate); + } + } + } + } + + return resultFFMatrixBlock; + } else { + std::cerr << "Wrong librayCode!" << std::endl; + exit(1); + } + }); + } +}; + +#endif diff --git a/src/FF/headers/FFTransposeBiasSum.h b/src/FF/headers/FFTransposeBiasSum.h index ad686efd..a216b212 100644 --- a/src/FF/headers/FFTransposeBiasSum.h +++ b/src/FF/headers/FFTransposeBiasSum.h @@ -1,103 +1,103 @@ -#ifndef FF_TRANSPOSE_BIAS_SUM_H -#define FF_TRANSPOSE_BIAS_SUM_H - -#include "FFMatrixBlock.h" -#include "JoinComp.h" -#include "Lambda.h" -#include "LambdaCreationFunctions.h" - -// LA libraries: -#include -#include - -using namespace pdb; - -class FFTransposeBiasSum - : public JoinComp { - -public: - ENABLE_DEEP_COPY - - FFTransposeBiasSum() = default; - - Lambda getSelection(Handle in1, - Handle in2) override { - // return makeLambda( - - // in1, in2, [](Handle &in1, Handle &in2) - // { - // return in1->getBlockRowIndex() == in2->getBlockRowIndex(); - // }); - return makeLambdaFromMethod(in1, getBlockRowIndex) == - makeLambdaFromMethod(in2, getBlockRowIndex); - - } - - Lambda> - getProjection(Handle in1, Handle in2) override { - return makeLambda( - in1, in2, [&](Handle &in1, Handle &in2) { - if (FFMatrixBlock::librayCode == EIGEN_CODE) { - // get the sizes - - uint32_t I = in1->getRowNums(); - uint32_t J = in1->getColNums(); - - if (in1->getRowNums() != in2->getRowNums() || - in2->getColNums() != 1) { - std::cerr << "Block dimemsions mismatch!" << std::endl; - exit(1); - } - - pdb::Handle resultFFMatrixBlock = - pdb::makeObject( - in1->getBlockColIndex(), in1->getBlockRowIndex(), J, I, - in1->getTotalColNums(), in1->getTotalRowNums(), false); - - double *outData = resultFFMatrixBlock->getValue().rawData->c_ptr(); - double *in1Data = in1->getValue().rawData->c_ptr(); - double *in2Data = in2->getValue().rawData->c_ptr(); - - Eigen::Map> - currentMatrix1(in1Data, I, J); - Eigen::Map< - Eigen::Matrix> - currentMatrix2(in2Data, I, 1); - - Eigen::Map> - sumMatrix(outData, I, J); - - sumMatrix = currentMatrix1.colwise() + currentMatrix2; - sumMatrix.transposeInPlace(); - - int row_idx = in1->getBlockColIndex(); - int col_idx = in1->getBlockRowIndex(); - int block_x = J; - int block_y = I; - int X = in1->getTotalColNums(); - int Y = in1->getTotalRowNums(); - - for (int i = 0; i < block_x; i++) { - int act_x = row_idx * block_x + i; - for (int j = 0; j < block_y; j++) { - int act_y = col_idx * block_y + j; - - if (act_x < X && act_y < Y) { - int cur_pos = i * block_x + j; - outData[cur_pos] = exp(outData[cur_pos]); - } - } - } - - return resultFFMatrixBlock; - } else { - std::cerr << "Wrong librayCode!" << std::endl; - exit(1); - } - }); - } -}; - +#ifndef FF_TRANSPOSE_BIAS_SUM_H +#define FF_TRANSPOSE_BIAS_SUM_H + +#include "FFMatrixBlock.h" +#include "JoinComp.h" +#include "Lambda.h" +#include "LambdaCreationFunctions.h" + +// LA libraries: +#include +#include + +using namespace pdb; + +class FFTransposeBiasSum + : public JoinComp { + +public: + ENABLE_DEEP_COPY + + FFTransposeBiasSum() = default; + + Lambda getSelection(Handle in1, + Handle in2) override { + // return makeLambda( + + // in1, in2, [](Handle &in1, Handle &in2) + // { + // return in1->getBlockRowIndex() == in2->getBlockRowIndex(); + // }); + return makeLambdaFromMethod(in1, getBlockRowIndex) == + makeLambdaFromMethod(in2, getBlockRowIndex); + + } + + Lambda> + getProjection(Handle in1, Handle in2) override { + return makeLambda( + in1, in2, [&](Handle &in1, Handle &in2) { + if (FFMatrixBlock::librayCode == EIGEN_CODE) { + // get the sizes + + uint32_t I = in1->getRowNums(); + uint32_t J = in1->getColNums(); + + if (in1->getRowNums() != in2->getRowNums() || + in2->getColNums() != 1) { + std::cerr << "Block dimemsions mismatch!" << std::endl; + exit(1); + } + + pdb::Handle resultFFMatrixBlock = + pdb::makeObject( + in1->getBlockColIndex(), in1->getBlockRowIndex(), J, I, + in1->getTotalColNums(), in1->getTotalRowNums(), false); + + float *outData = resultFFMatrixBlock->getValue().rawData->c_ptr(); + float *in1Data = in1->getValue().rawData->c_ptr(); + float *in2Data = in2->getValue().rawData->c_ptr(); + + Eigen::Map> + currentMatrix1(in1Data, I, J); + Eigen::Map< + Eigen::Matrix> + currentMatrix2(in2Data, I, 1); + + Eigen::Map> + sumMatrix(outData, I, J); + + sumMatrix = currentMatrix1.colwise() + currentMatrix2; + sumMatrix.transposeInPlace(); + + int row_idx = in1->getBlockColIndex(); + int col_idx = in1->getBlockRowIndex(); + int block_x = J; + int block_y = I; + int X = in1->getTotalColNums(); + int Y = in1->getTotalRowNums(); + + for (int i = 0; i < block_x; i++) { + int act_x = row_idx * block_x + i; + for (int j = 0; j < block_y; j++) { + int act_y = col_idx * block_y + j; + + if (act_x < X && act_y < Y) { + int cur_pos = i * block_x + j; + outData[cur_pos] = exp(outData[cur_pos]); + } + } + } + + return resultFFMatrixBlock; + } else { + std::cerr << "Wrong librayCode!" << std::endl; + exit(1); + } + }); + } +}; + #endif \ No newline at end of file diff --git a/src/FF/headers/FFTransposeMult.h b/src/FF/headers/FFTransposeMult.h index 2cc3947a..066dd426 100644 --- a/src/FF/headers/FFTransposeMult.h +++ b/src/FF/headers/FFTransposeMult.h @@ -1,81 +1,81 @@ -#ifndef FF_TRANSPOSE_MULT_H -#define FF_TRANSPOSE_MULT_H - -#include "FFMatrixBlock.h" -#include "JoinComp.h" -#include "Lambda.h" -#include "LambdaCreationFunctions.h" - -// LA libraries: -#include - -using namespace pdb; - -class FFTransposeMult - : public JoinComp { - -public: - ENABLE_DEEP_COPY - - FFTransposeMult() = default; - - Lambda getSelection(Handle in1, - Handle in2) override { - // return makeLambda( - - // in1, in2, [](Handle &in1, Handle &in2) { - // return in1->getBlockColIndex() == in2->getBlockColIndex(); - // }); - // Only this can help with partitioning - return makeLambdaFromMethod(in1, getBlockColIndex) == - makeLambdaFromMethod(in2, getBlockColIndex); - - } - - Lambda> - getProjection(Handle in1, Handle in2) override { - return makeLambda( - in1, in2, [&](Handle &in1, Handle &in2) { - if (FFMatrixBlock::librayCode == EIGEN_CODE) { - // get the sizes - uint32_t I = in1->getRowNums(); - uint32_t J = in2->getRowNums(); - - if (in1->getColNums() != in2->getColNums()) { - std::cerr << "Block dimemsions mismatch! " << std::endl; - exit(1); - } - - pdb::Handle resultFFMatrixBlock = - pdb::makeObject( - in1->getBlockRowIndex(), in2->getBlockRowIndex(), I, J, - in1->getTotalRowNums(), in2->getTotalRowNums(), false); - - // get the ptrs - double *outData = resultFFMatrixBlock->getValue().rawData->c_ptr(); - double *in1Data = in1->getValue().rawData->c_ptr(); - double *in2Data = in2->getValue().rawData->c_ptr(); - - Eigen::Map> - currentMatrix1(in1Data, in1->getRowNums(), in1->getColNums()); - Eigen::Map> - currentMatrix2(in2Data, in2->getRowNums(), in2->getColNums()); - - Eigen::Map> - productMatrix(outData, I, J); - - productMatrix = currentMatrix1 * currentMatrix2.transpose(); - - return resultFFMatrixBlock; - } else { - std::cerr << "Wrong librayCode!" << std::endl; - exit(1); - } - }); - } -}; - -#endif +#ifndef FF_TRANSPOSE_MULT_H +#define FF_TRANSPOSE_MULT_H + +#include "FFMatrixBlock.h" +#include "JoinComp.h" +#include "Lambda.h" +#include "LambdaCreationFunctions.h" + +// LA libraries: +#include + +using namespace pdb; + +class FFTransposeMult + : public JoinComp { + +public: + ENABLE_DEEP_COPY + + FFTransposeMult() = default; + + Lambda getSelection(Handle in1, + Handle in2) override { + // return makeLambda( + + // in1, in2, [](Handle &in1, Handle &in2) { + // return in1->getBlockColIndex() == in2->getBlockColIndex(); + // }); + // Only this can help with partitioning + return makeLambdaFromMethod(in1, getBlockColIndex) == + makeLambdaFromMethod(in2, getBlockColIndex); + + } + + Lambda> + getProjection(Handle in1, Handle in2) override { + return makeLambda( + in1, in2, [&](Handle &in1, Handle &in2) { + if (FFMatrixBlock::librayCode == EIGEN_CODE) { + // get the sizes + uint32_t I = in1->getRowNums(); + uint32_t J = in2->getRowNums(); + + if (in1->getColNums() != in2->getColNums()) { + std::cerr << "Block dimemsions mismatch! " << std::endl; + exit(1); + } + + pdb::Handle resultFFMatrixBlock = + pdb::makeObject( + in1->getBlockRowIndex(), in2->getBlockRowIndex(), I, J, + in1->getTotalRowNums(), in2->getTotalRowNums(), false); + + // get the ptrs + float *outData = resultFFMatrixBlock->getValue().rawData->c_ptr(); + float *in1Data = in1->getValue().rawData->c_ptr(); + float *in2Data = in2->getValue().rawData->c_ptr(); + + Eigen::Map> + currentMatrix1(in1Data, in1->getRowNums(), in1->getColNums()); + Eigen::Map> + currentMatrix2(in2Data, in2->getRowNums(), in2->getColNums()); + + Eigen::Map> + productMatrix(outData, I, J); + + productMatrix = currentMatrix1 * currentMatrix2.transpose(); + + return resultFFMatrixBlock; + } else { + std::cerr << "Wrong librayCode!" << std::endl; + exit(1); + } + }); + } +}; + +#endif diff --git a/src/FF/headers/InferenceResult.h b/src/FF/headers/InferenceResult.h index 7b2f9a81..f5edbd5d 100644 --- a/src/FF/headers/InferenceResult.h +++ b/src/FF/headers/InferenceResult.h @@ -23,7 +23,7 @@ class InferenceResult : public Object { int index; int block_row_id; - Handle> inference = nullptr; + Handle> inference = nullptr; ENABLE_DEEP_COPY @@ -32,12 +32,12 @@ class InferenceResult : public Object { InferenceResult(int index, int block_row_id) : index(index), block_row_id(block_row_id) { - inference = makeObject>(); + inference = makeObject>(); } int getKey() { return this->index; } - Vector &getInference() { return (*inference); } + Vector &getInference() { return (*inference); } int getLabel() { return (*inference)[0] > (*inference)[1] ? 1 : -1; } diff --git a/src/FF/source/FFMatrixUtil.cc b/src/FF/source/FFMatrixUtil.cc index 73722974..c46f0fe6 100644 --- a/src/FF/source/FFMatrixUtil.cc +++ b/src/FF/source/FFMatrixUtil.cc @@ -1,481 +1,481 @@ -#include "FFMatrixUtil.h" - -#include "FFMatrixBlock.h" - -#include "PDBClient.h" -#include -#include -using namespace std; - -namespace ff { -void load_matrix_data(pdb::PDBClient &pdbClient, string path, - pdb::String dbName, pdb::String setName, int blockX, - int blockY, bool dont_pad_x, bool dont_pad_y, - string &errMsg, int size, bool partitionByCol) { - if (path.size() == 0) { - throw runtime_error("Invalid filepath: " + path); - } - - int totalX, totalY; - - /// 1. Load the data from the file - - // open the input file - ifstream is(path); - while (is.peek() == '#' || is.peek() == ' ') - is.ignore(); - - // load the data stats - is >> totalX; - - while (is.peek() == ',' || is.peek() == ' ') - is.ignore(); - - is >> totalY; - - cout << totalX << ", " << totalY << endl; - - vector> matrix; - - double val; - int total = 0; - pdb::makeObjectAllocatorBlock(size * 1024 * 1024, true); - - pdb::Handle>> storeMatrix1 = - pdb::makeObject>>(); - - int numXBlocks = ceil(totalX / (double)blockX); - int numYBlocks = ceil(totalY / (double)blockY); - - int i = 0; - int j = 0; - int ii = 0; - int jj = 0; - - while (i < numXBlocks) { - - try { - while (i < numXBlocks) { - int actual_blockX = - dont_pad_x ? min(blockX, totalX - i * blockX) : blockX; - - while (j < numYBlocks) { - int actual_blockY = - dont_pad_y ? min(blockY, totalY - j * blockY) : blockY; - - pdb::Handle myData = - pdb::makeObject(i, j, actual_blockX, actual_blockY, - totalX, totalY, partitionByCol); - - while (ii < actual_blockX) { - while (jj < actual_blockY) { - int curX = (i * actual_blockX + ii); - int curY = (j * actual_blockY + jj); - - if ((dont_pad_x && curX >= totalX) || - (dont_pad_y && curY >= totalY)) { - cout << "Shouldnt be here!" << endl; - exit(1); - } - - is >> val; - while (is.peek() == ',' || is.peek() == ' ') - is.ignore(); - - // row = i * blockX + ii, col = j * blockY + jj - double data = curX >= totalX || curY >= totalY ? 0 : val; - (*(myData->getRawDataHandle()))[ii * actual_blockY + jj] = data; - jj++; - } - ii++; - jj = 0; - } - - // cout << "New block: " << total << endl; - storeMatrix1->push_back(myData); - total++; - j++; - ii = 0; - jj = 0; - } - - i++; - j = 0; - ii = 0; - jj = 0; - } - if (!pdbClient.sendData( - pair(setName, dbName), storeMatrix1, errMsg)) { - cout << "Failed to send data to dispatcher server" << endl; - exit(1); - } - } catch (pdb::NotEnoughSpace &e) { - if (!pdbClient.sendData( - pair(setName, dbName), storeMatrix1, errMsg)) { - cout << "Failed to send data to dispatcher server" << endl; - exit(1); - } - std::cout << "Dispatched " << storeMatrix1->size() << " blocks." - << std::endl; - pdb::makeObjectAllocatorBlock(size * 1024 * 1024, true); - storeMatrix1 = pdb::makeObject>>(); - } - } - - cout << setName << "(" << totalX << "x" << totalY << "): (" << numXBlocks - << " x " << numYBlocks << ")" << total << " blocks = " << blockX << " x " - << blockY << " each" << endl; - - // to write back all buffered records - pdbClient.flushData(errMsg); -} - -void load_matrix_data(pdb::PDBClient &pdbClient, string path, - pdb::String dbName, pdb::String setName, int pad_x, - int pad_y, string &errMsg) { - if (path.size() == 0) { - throw runtime_error("Invalid filepath: " + path); - } - - int totalX, totalY; - - /// 1. Load the data from the file - - // open the input file - ifstream is(path); - while (is.peek() == '#' || is.peek() == ' ') - is.ignore(); - - // load the data stats - is >> totalX; - - while (is.peek() == ',' || is.peek() == ' ') - is.ignore(); - - is >> totalY; - - cout << totalX << ", " << totalY << endl; - - vector> matrix; - - double val; - for (int i = 0; i < totalX; i++) { - vector line; - for (int j = 0; j < totalY; j++) { - is >> val; - line.push_back(val); - while (is.peek() == ',' || is.peek() == ' ') - is.ignore(); - } - matrix.push_back(line); - } - - if (matrix.size() == 0) { - throw runtime_error("Invalid matrix file: " + path); - } - - int total = 0; - pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); - - pdb::Handle>> storeMatrix1 = - pdb::makeObject>>(); - - try { - pdb::Handle myData = pdb::makeObject( - 0, 0, totalX + pad_x, totalY + pad_y, totalX + pad_x, totalY + pad_y); - - for (int i = 0; i < totalX + pad_x; i++) { - for (int j = 0; j < totalY + pad_y; j++) { - double data = i >= totalX || j >= totalY - ? 0 // padding to adjust to dimensions - : matrix[i][j]; - (*(myData->getRawDataHandle()))[i * (totalY + pad_y) + j] = data; - } - } - - storeMatrix1->push_back(myData); - - if (!pdbClient.sendData( - pair(setName, dbName), storeMatrix1, errMsg)) { - cout << "Failed to send data to dispatcher server" << endl; - exit(1); - } - } catch (pdb::NotEnoughSpace &e) { - cout << "Failed to send data to dispatcher server" << endl; - exit(1); - } - - // to write back all buffered records - pdbClient.flushData(errMsg); -} - -void loadMatrix(pdb::PDBClient &pdbClient, pdb::String dbName, - pdb::String setName, int totalX, int totalY, int blockX, - int blockY, bool dont_pad_x, bool dont_pad_y, string &errMsg, - int size, bool partitionByCol) { - - if ((totalX == 0) || (totalY == 0) || (blockX == 0) || (blockY == 0)) { - - return; - - } - - std::cout << "totalX=" << totalX << ", totalY=" << totalY - << ",blockX=" << blockX << ", blockY=" << blockY << std::endl; - std::random_device rd; - - std::mt19937 e2(rd()); - - std::uniform_real_distribution<> distp(0.0001, 0.5); - std::uniform_real_distribution<> distn(-0.5, -0.0001); - - auto gen = std::bind(std::uniform_int_distribution<>(0, 1), - std::default_random_engine()); - - int total = 0; - pdb::makeObjectAllocatorBlock(size * 1024 * 1024, true); - - pdb::Handle>> storeMatrix1 = - pdb::makeObject>>(); - - int numXBlocks = ceil(totalX / (double)blockX); - int numYBlocks = ceil(totalY / (double)blockY); - - int i = 0; - int j = 0; - int ii = 0; - int jj = 0; - - while (i < numXBlocks) { - - try { - while (i < numXBlocks) { - int actual_blockX = - dont_pad_x ? min(blockX, totalX - i * blockX) : blockX; - - while (j < numYBlocks) { - int actual_blockY = - dont_pad_y ? min(blockY, totalY - j * blockY) : blockY; - - pdb::Handle myData = - pdb::makeObject(i, j, actual_blockX, actual_blockY, - totalX, totalY, partitionByCol); - - while (ii < actual_blockX) { - while (jj < actual_blockY) { - int curX = (i * actual_blockX + ii); - int curY = (j * actual_blockY + jj); - - if ((dont_pad_x && curX >= totalX) || - (dont_pad_y && curY >= totalY)) { - cout << "Shouldnt be here!" << endl; - exit(1); - } - - // row = i * blockX + ii, col = j * blockY + jj - double data = curX >= totalX || curY >= totalY ? 0 - : (bool)gen() ? distn(e2) - : distp(e2); - (*(myData->getRawDataHandle()))[ii * actual_blockY + jj] = data; - jj++; - } - ii++; - jj = 0; - } - - std::cout << "New block: " << total << endl; - storeMatrix1->push_back(myData); - total++; - j++; - ii = 0; - jj = 0; - } - - i++; - j = 0; - ii = 0; - jj = 0; - } - if (!pdbClient.sendData( - pair(setName, dbName), storeMatrix1, errMsg)) { - cout << "Failed to send data to dispatcher server" << endl; - exit(1); - } - } catch (pdb::NotEnoughSpace &e) { - if (!pdbClient.sendData( - pair(setName, dbName), storeMatrix1, errMsg)) { - cout << "Failed to send data to dispatcher server" << endl; - exit(1); - } - std::cout << "Dispatched " << storeMatrix1->size() << " blocks." - << std::endl; - pdb::makeObjectAllocatorBlock(size * 1024 * 1024, true); - storeMatrix1 = pdb::makeObject>>(); - } - } - - cout << setName << "(" << totalX << "x" << totalY << "): (" << numXBlocks - << " x " << numYBlocks << ")" << total << " blocks = " << blockX << " x " - << blockY << " each" << endl; - - // to write back all buffered records - pdbClient.flushData(errMsg); -} - -void load_matrix_from_file(string path, vector> &matrix) { - if (path.size() == 0) { - throw runtime_error("Invalid filepath: " + path); - } - - int totalX, totalY; - - /// 1. Load the data from the file - - // open the input file - ifstream is(path); - while (is.peek() == '#' || is.peek() == ' ') - is.ignore(); - - // load the data stats - is >> totalX; - - while (is.peek() == ',' || is.peek() == ' ') - is.ignore(); - - is >> totalY; - - cout << totalX << ", " << totalY << endl; - - double val; - for (int i = 0; i < totalX; i++) { - vector line; - for (int j = 0; j < totalY; j++) { - is >> val; - line.push_back(val); - while (is.peek() == ',' || is.peek() == ' ') - is.ignore(); - } - matrix.push_back(line); - } -} - -bool is_empty_set(pdb::PDBClient &pdbClient, pdb::CatalogClient &catalogClient, - string dbName, string setName) { - if (!catalogClient.setExists(dbName, setName)) - return true; - - auto it = pdbClient.getSetIterator(dbName, setName); - int count = 0; - for (auto r : it) { - count++; - } - - if (count != 0) - cout << "[VALIDATE] Set " << setName << " exists!" << endl; - - return count == 0; -} - -void print(pdb::PDBClient &pdbClient, string dbName, string setName) { - auto it = pdbClient.getSetIterator(dbName, setName); - bool first = true; - int count = 0; - for (auto r : it) { - double *ndata = r->getRawDataHandle()->c_ptr(); - int x = r->getBlockRowIndex(); - int y = r->getBlockColIndex(); - int bx = r->getRowNums(); - int by = r->getColNums(); - - if (first) { - first = false; - cout << "[STATS] " << setName << " : " << r->getTotalRowNums() << " X " << r->getTotalColNums() << endl; - } - - /*cout << "[PRINT] " << setName << " : " << x << "," << y - << "; Block Size: " << bx << "," << by << endl; - for (int i = 0; i < bx; i++) { - for (int j = 0; j < by; j++) { - cout << ndata[i * by + j] << ","; - } - cout << endl; - } - cout << endl;*/ - count++; - } - std::cout << "count = " << count << std::endl; -} - -void print_old(pdb::PDBClient &pdbClient, string dbName, string setName) { - pdb::Map>> data; - auto it = pdbClient.getSetIterator(dbName, setName); - - int max_rows = 0, max_cols = 0; - int ax = 0, ay = 0, bx = 0, by = 0; - - for (auto r : it) { - int x = r->getBlockRowIndex(); - int y = r->getBlockColIndex(); - max_rows = max_rows < x ? x : max_rows; - max_cols = max_cols < y ? y : max_cols; - ax = r->getTotalRowNums(); - ay = r->getTotalColNums(); - bx = r->getRowNums(); - by = r->getColNums(); - - if (data.count(x) != 0 || data[x].count(y) != 0) - cout << "[WARNING!] Already have " << x << "," << y << " block for " - << setName << endl; - - data[x][y] = r; - } - - long count = 0; - - cout << setName << ": "; - cout << "Actual dimensions: (" << ax << ", " << ay << "), " - << "; Block size: (" << bx << ", " << by << ")" << endl; - - for (int i = 0; i <= max_rows; i++) { - for (int j = 0; j < bx; j++) { - for (int k = 0; k <= max_cols; k++) { - for (int l = 0; l < by; l++) { - if (data[i][k]->getRawDataHandle() == nullptr) - continue; - double *ndata = data[i][k]->getRawDataHandle()->c_ptr(); - int pos = j * by + l; - cout << ndata[pos] << ","; - } - } - cout << endl; - } - // One row of blocks done - // cout << endl; - } -} - -void print_stats(pdb::PDBClient &pdbClient, string dbName, string setName) { - int rows = 0, cols = 0, blocks = 0; - int totalRows = 0, totalCols = 0; - int blockRows = 0, blockCols = 0; - auto it = pdbClient.getSetIterator(dbName, setName); - - cout << "[STATS]: " << setName << endl; - for (auto r : it) { - // cout << r->getBlockRowIndex() << "," << r->getBlockColIndex() << ";"; - rows = r->getRowNums(); - cols = r->getColNums(); - totalRows = r->getTotalRowNums(); - totalCols = r->getTotalColNums(); - blockRows = max(r->getBlockRowIndex(), blockRows); - blockCols = max(r->getBlockColIndex(), blockCols); - blocks++; - } - - cout << "\n" - << setName << " (" << (blockRows + 1) << " X " << (blockCols + 1) - << ") (" << blocks << ") : (" << totalRows << " x " << totalCols - << "), Each block size: " << rows << " x " << cols << endl; -} - -} // namespace ff +#include "FFMatrixUtil.h" + +#include "FFMatrixBlock.h" + +#include "PDBClient.h" +#include +#include +using namespace std; + +namespace ff { +void load_matrix_data(pdb::PDBClient &pdbClient, string path, + pdb::String dbName, pdb::String setName, int blockX, + int blockY, bool dont_pad_x, bool dont_pad_y, + string &errMsg, int size, bool partitionByCol) { + if (path.size() == 0) { + throw runtime_error("Invalid filepath: " + path); + } + + int totalX, totalY; + + /// 1. Load the data from the file + + // open the input file + ifstream is(path); + while (is.peek() == '#' || is.peek() == ' ') + is.ignore(); + + // load the data stats + is >> totalX; + + while (is.peek() == ',' || is.peek() == ' ') + is.ignore(); + + is >> totalY; + + cout << totalX << ", " << totalY << endl; + + vector> matrix; + + float val; + int total = 0; + pdb::makeObjectAllocatorBlock(size * 1024 * 1024, true); + + pdb::Handle>> storeMatrix1 = + pdb::makeObject>>(); + + int numXBlocks = ceil(totalX / (double)blockX); + int numYBlocks = ceil(totalY / (double)blockY); + + int i = 0; + int j = 0; + int ii = 0; + int jj = 0; + + while (i < numXBlocks) { + + try { + while (i < numXBlocks) { + int actual_blockX = + dont_pad_x ? min(blockX, totalX - i * blockX) : blockX; + + while (j < numYBlocks) { + int actual_blockY = + dont_pad_y ? min(blockY, totalY - j * blockY) : blockY; + + pdb::Handle myData = + pdb::makeObject(i, j, actual_blockX, actual_blockY, + totalX, totalY, partitionByCol); + + while (ii < actual_blockX) { + while (jj < actual_blockY) { + int curX = (i * actual_blockX + ii); + int curY = (j * actual_blockY + jj); + + if ((dont_pad_x && curX >= totalX) || + (dont_pad_y && curY >= totalY)) { + cout << "Shouldnt be here!" << endl; + exit(1); + } + + is >> val; + while (is.peek() == ',' || is.peek() == ' ') + is.ignore(); + + // row = i * blockX + ii, col = j * blockY + jj + float data = curX >= totalX || curY >= totalY ? 0 : val; + (*(myData->getRawDataHandle()))[ii * actual_blockY + jj] = data; + jj++; + } + ii++; + jj = 0; + } + + // cout << "New block: " << total << endl; + storeMatrix1->push_back(myData); + total++; + j++; + ii = 0; + jj = 0; + } + + i++; + j = 0; + ii = 0; + jj = 0; + } + if (!pdbClient.sendData( + pair(setName, dbName), storeMatrix1, errMsg)) { + cout << "Failed to send data to dispatcher server" << endl; + exit(1); + } + } catch (pdb::NotEnoughSpace &e) { + if (!pdbClient.sendData( + pair(setName, dbName), storeMatrix1, errMsg)) { + cout << "Failed to send data to dispatcher server" << endl; + exit(1); + } + std::cout << "Dispatched " << storeMatrix1->size() << " blocks." + << std::endl; + pdb::makeObjectAllocatorBlock(size * 1024 * 1024, true); + storeMatrix1 = pdb::makeObject>>(); + } + } + + cout << setName << "(" << totalX << "x" << totalY << "): (" << numXBlocks + << " x " << numYBlocks << ")" << total << " blocks = " << blockX << " x " + << blockY << " each" << endl; + + // to write back all buffered records + pdbClient.flushData(errMsg); +} + +void load_matrix_data(pdb::PDBClient &pdbClient, string path, + pdb::String dbName, pdb::String setName, int pad_x, + int pad_y, string &errMsg) { + if (path.size() == 0) { + throw runtime_error("Invalid filepath: " + path); + } + + int totalX, totalY; + + /// 1. Load the data from the file + + // open the input file + ifstream is(path); + while (is.peek() == '#' || is.peek() == ' ') + is.ignore(); + + // load the data stats + is >> totalX; + + while (is.peek() == ',' || is.peek() == ' ') + is.ignore(); + + is >> totalY; + + cout << totalX << ", " << totalY << endl; + + vector> matrix; + + float val; + for (int i = 0; i < totalX; i++) { + vector line; + for (int j = 0; j < totalY; j++) { + is >> val; + line.push_back(val); + while (is.peek() == ',' || is.peek() == ' ') + is.ignore(); + } + matrix.push_back(line); + } + + if (matrix.size() == 0) { + throw runtime_error("Invalid matrix file: " + path); + } + + int total = 0; + pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); + + pdb::Handle>> storeMatrix1 = + pdb::makeObject>>(); + + try { + pdb::Handle myData = pdb::makeObject( + 0, 0, totalX + pad_x, totalY + pad_y, totalX + pad_x, totalY + pad_y); + + for (int i = 0; i < totalX + pad_x; i++) { + for (int j = 0; j < totalY + pad_y; j++) { + float data = i >= totalX || j >= totalY + ? 0 // padding to adjust to dimensions + : matrix[i][j]; + (*(myData->getRawDataHandle()))[i * (totalY + pad_y) + j] = data; + } + } + + storeMatrix1->push_back(myData); + + if (!pdbClient.sendData( + pair(setName, dbName), storeMatrix1, errMsg)) { + cout << "Failed to send data to dispatcher server" << endl; + exit(1); + } + } catch (pdb::NotEnoughSpace &e) { + cout << "Failed to send data to dispatcher server" << endl; + exit(1); + } + + // to write back all buffered records + pdbClient.flushData(errMsg); +} + +void loadMatrix(pdb::PDBClient &pdbClient, pdb::String dbName, + pdb::String setName, int totalX, int totalY, int blockX, + int blockY, bool dont_pad_x, bool dont_pad_y, string &errMsg, + int size, bool partitionByCol) { + + if ((totalX == 0) || (totalY == 0) || (blockX == 0) || (blockY == 0)) { + + return; + + } + + std::cout << "totalX=" << totalX << ", totalY=" << totalY + << ",blockX=" << blockX << ", blockY=" << blockY << std::endl; + std::random_device rd; + + std::mt19937 e2(rd()); + + std::uniform_real_distribution<> distp(0.0001, 0.5); + std::uniform_real_distribution<> distn(-0.5, -0.0001); + + auto gen = std::bind(std::uniform_int_distribution<>(0, 1), + std::default_random_engine()); + + int total = 0; + pdb::makeObjectAllocatorBlock(size * 1024 * 1024, true); + + pdb::Handle>> storeMatrix1 = + pdb::makeObject>>(); + + int numXBlocks = ceil(totalX / (double)blockX); + int numYBlocks = ceil(totalY / (double)blockY); + + int i = 0; + int j = 0; + int ii = 0; + int jj = 0; + + while (i < numXBlocks) { + + try { + while (i < numXBlocks) { + int actual_blockX = + dont_pad_x ? min(blockX, totalX - i * blockX) : blockX; + + while (j < numYBlocks) { + int actual_blockY = + dont_pad_y ? min(blockY, totalY - j * blockY) : blockY; + + pdb::Handle myData = + pdb::makeObject(i, j, actual_blockX, actual_blockY, + totalX, totalY, partitionByCol); + + while (ii < actual_blockX) { + while (jj < actual_blockY) { + int curX = (i * actual_blockX + ii); + int curY = (j * actual_blockY + jj); + + if ((dont_pad_x && curX >= totalX) || + (dont_pad_y && curY >= totalY)) { + cout << "Shouldnt be here!" << endl; + exit(1); + } + + // row = i * blockX + ii, col = j * blockY + jj + float data = curX >= totalX || curY >= totalY ? 0 + : (bool)gen() ? distn(e2) + : distp(e2); + (*(myData->getRawDataHandle()))[ii * actual_blockY + jj] = data; + jj++; + } + ii++; + jj = 0; + } + + std::cout << "New block: " << total << endl; + storeMatrix1->push_back(myData); + total++; + j++; + ii = 0; + jj = 0; + } + + i++; + j = 0; + ii = 0; + jj = 0; + } + if (!pdbClient.sendData( + pair(setName, dbName), storeMatrix1, errMsg)) { + cout << "Failed to send data to dispatcher server" << endl; + exit(1); + } + } catch (pdb::NotEnoughSpace &e) { + if (!pdbClient.sendData( + pair(setName, dbName), storeMatrix1, errMsg)) { + cout << "Failed to send data to dispatcher server" << endl; + exit(1); + } + std::cout << "Dispatched " << storeMatrix1->size() << " blocks." + << std::endl; + pdb::makeObjectAllocatorBlock(size * 1024 * 1024, true); + storeMatrix1 = pdb::makeObject>>(); + } + } + + cout << setName << "(" << totalX << "x" << totalY << "): (" << numXBlocks + << " x " << numYBlocks << ")" << total << " blocks = " << blockX << " x " + << blockY << " each" << endl; + + // to write back all buffered records + pdbClient.flushData(errMsg); +} + +void load_matrix_from_file(string path, vector> &matrix) { + if (path.size() == 0) { + throw runtime_error("Invalid filepath: " + path); + } + + int totalX, totalY; + + /// 1. Load the data from the file + + // open the input file + ifstream is(path); + while (is.peek() == '#' || is.peek() == ' ') + is.ignore(); + + // load the data stats + is >> totalX; + + while (is.peek() == ',' || is.peek() == ' ') + is.ignore(); + + is >> totalY; + + cout << totalX << ", " << totalY << endl; + + float val; + for (int i = 0; i < totalX; i++) { + vector line; + for (int j = 0; j < totalY; j++) { + is >> val; + line.push_back(val); + while (is.peek() == ',' || is.peek() == ' ') + is.ignore(); + } + matrix.push_back(line); + } +} + +bool is_empty_set(pdb::PDBClient &pdbClient, pdb::CatalogClient &catalogClient, + string dbName, string setName) { + if (!catalogClient.setExists(dbName, setName)) + return true; + + auto it = pdbClient.getSetIterator(dbName, setName); + int count = 0; + for (auto r : it) { + count++; + } + + if (count != 0) + cout << "[VALIDATE] Set " << setName << " exists!" << endl; + + return count == 0; +} + +void print(pdb::PDBClient &pdbClient, string dbName, string setName) { + auto it = pdbClient.getSetIterator(dbName, setName); + bool first = true; + int count = 0; + for (auto r : it) { + float *ndata = r->getRawDataHandle()->c_ptr(); + int x = r->getBlockRowIndex(); + int y = r->getBlockColIndex(); + int bx = r->getRowNums(); + int by = r->getColNums(); + + if (first) { + first = false; + cout << "[STATS] " << setName << " : " << r->getTotalRowNums() << " X " << r->getTotalColNums() << endl; + } + + /*cout << "[PRINT] " << setName << " : " << x << "," << y + << "; Block Size: " << bx << "," << by << endl; + for (int i = 0; i < bx; i++) { + for (int j = 0; j < by; j++) { + cout << ndata[i * by + j] << ","; + } + cout << endl; + } + cout << endl;*/ + count++; + } + std::cout << "count = " << count << std::endl; +} + +void print_old(pdb::PDBClient &pdbClient, string dbName, string setName) { + pdb::Map>> data; + auto it = pdbClient.getSetIterator(dbName, setName); + + int max_rows = 0, max_cols = 0; + int ax = 0, ay = 0, bx = 0, by = 0; + + for (auto r : it) { + int x = r->getBlockRowIndex(); + int y = r->getBlockColIndex(); + max_rows = max_rows < x ? x : max_rows; + max_cols = max_cols < y ? y : max_cols; + ax = r->getTotalRowNums(); + ay = r->getTotalColNums(); + bx = r->getRowNums(); + by = r->getColNums(); + + if (data.count(x) != 0 || data[x].count(y) != 0) + cout << "[WARNING!] Already have " << x << "," << y << " block for " + << setName << endl; + + data[x][y] = r; + } + + long count = 0; + + cout << setName << ": "; + cout << "Actual dimensions: (" << ax << ", " << ay << "), " + << "; Block size: (" << bx << ", " << by << ")" << endl; + + for (int i = 0; i <= max_rows; i++) { + for (int j = 0; j < bx; j++) { + for (int k = 0; k <= max_cols; k++) { + for (int l = 0; l < by; l++) { + if (data[i][k]->getRawDataHandle() == nullptr) + continue; + float *ndata = data[i][k]->getRawDataHandle()->c_ptr(); + int pos = j * by + l; + cout << ndata[pos] << ","; + } + } + cout << endl; + } + // One row of blocks done + // cout << endl; + } +} + +void print_stats(pdb::PDBClient &pdbClient, string dbName, string setName) { + int rows = 0, cols = 0, blocks = 0; + int totalRows = 0, totalCols = 0; + int blockRows = 0, blockCols = 0; + auto it = pdbClient.getSetIterator(dbName, setName); + + cout << "[STATS]: " << setName << endl; + for (auto r : it) { + // cout << r->getBlockRowIndex() << "," << r->getBlockColIndex() << ";"; + rows = r->getRowNums(); + cols = r->getColNums(); + totalRows = r->getTotalRowNums(); + totalCols = r->getTotalColNums(); + blockRows = max(r->getBlockRowIndex(), blockRows); + blockCols = max(r->getBlockColIndex(), blockCols); + blocks++; + } + + cout << "\n" + << setName << " (" << (blockRows + 1) << " X " << (blockCols + 1) + << ") (" << blocks << ") : (" << totalRows << " x " << totalCols + << "), Each block size: " << rows << " x " << cols << endl; +} + +} // namespace ff diff --git a/src/conf/headers/Configuration.h b/src/conf/headers/Configuration.h index 2521f493..ac20ddaf 100644 --- a/src/conf/headers/Configuration.h +++ b/src/conf/headers/Configuration.h @@ -24,7 +24,7 @@ using namespace std; #endif #ifndef DEFAULT_MAX_PAGE_SIZE -#define DEFAULT_MAX_PAGE_SIZE ((size_t)(1024) * (size_t)(1024) * (size_t)(1024)) +#define DEFAULT_MAX_PAGE_SIZE ((size_t)(2047) * (size_t)(1024) * (size_t)(1024)) #endif #ifndef DEFAULT_NET_PAGE_SIZE diff --git a/src/conv2d_memory_fusion/headers/ConvChunksToImage.h b/src/conv2d_memory_fusion/headers/ConvChunksToImage.h index 5fbe73cc..f9d29972 100644 --- a/src/conv2d_memory_fusion/headers/ConvChunksToImage.h +++ b/src/conv2d_memory_fusion/headers/ConvChunksToImage.h @@ -1,78 +1,78 @@ -#ifndef CONV2D_CONV_CHUNKS_TO_IMAGE_H -#define CONV2D_CONV_CHUNKS_TO_IMAGE_H - -#include -#include - -#include "ClusterAggregateComp.h" - -#include "Lambda.h" -#include "LambdaCreationFunctions.h" - -#include "PDBString.h" -#include "PDBVector.h" - -#include "ImageBlock.h" -#include "Image.h" -#include "FFMatrixBlock.h" - -using namespace pdb; - -namespace conv2d_memory_fusion { - -class ConvChunksToImage - : public ClusterAggregateComp { - -public: - ENABLE_DEEP_COPY - - int x; - int y; - int z; - - ConvChunksToImage() {} - - ConvChunksToImage(int x, int y, int z) : x(x), y(y), z(z) {} - - // the key type must have == and size_t hash () defined - Lambda getKeyProjection(Handle aggMe) override { - return makeLambda(aggMe, [this](Handle &aggMe) { - return (long) aggMe->block_x_index; - }); - } - - // the value type must have + defined - Lambda getValueProjection(Handle aggMe) override { - return makeLambda(aggMe, [this](Handle &aggMe) { - Handle temp = makeObject(aggMe->block_x_index, x, y, z); - Handle channel = makeObject(0, 0, x, y, x, y); - double *data = channel->getValue().rawData->c_ptr(); - - Map> &chunks = aggMe->getBlock(); - int i = 0; - while (i < x * y) { - assert(chunks.count(i) != 0); - Vector &in = chunks[i]; - - for(int j = 0; j < in.size(); j++) { - data[i + j] = in[j]; - } - - i += in.size(); - } - - for (int c = 0; c < z; c++) { - if (c == aggMe->block_y_index) - temp->addChannel(channel); - else - temp->addChannel(nullptr); - } - - return *temp; - }); - } -}; - -} // namespace conv2d_memory_fusion - -#endif +#ifndef CONV2D_CONV_CHUNKS_TO_IMAGE_H +#define CONV2D_CONV_CHUNKS_TO_IMAGE_H + +#include +#include + +#include "ClusterAggregateComp.h" + +#include "Lambda.h" +#include "LambdaCreationFunctions.h" + +#include "PDBString.h" +#include "PDBVector.h" + +#include "ImageBlock.h" +#include "Image.h" +#include "FFMatrixBlock.h" + +using namespace pdb; + +namespace conv2d_memory_fusion { + +class ConvChunksToImage + : public ClusterAggregateComp { + +public: + ENABLE_DEEP_COPY + + int x; + int y; + int z; + + ConvChunksToImage() {} + + ConvChunksToImage(int x, int y, int z) : x(x), y(y), z(z) {} + + // the key type must have == and size_t hash () defined + Lambda getKeyProjection(Handle aggMe) override { + return makeLambda(aggMe, [this](Handle &aggMe) { + return (long) aggMe->block_x_index; + }); + } + + // the value type must have + defined + Lambda getValueProjection(Handle aggMe) override { + return makeLambda(aggMe, [this](Handle &aggMe) { + Handle temp = makeObject(aggMe->block_x_index, x, y, z); + Handle channel = makeObject(0, 0, x, y, x, y); + float *data = channel->getValue().rawData->c_ptr(); + + Map> &chunks = aggMe->getBlock(); + int i = 0; + while (i < x * y) { + assert(chunks.count(i) != 0); + Vector &in = chunks[i]; + + for(int j = 0; j < in.size(); j++) { + data[i + j] = in[j]; + } + + i += in.size(); + } + + for (int c = 0; c < z; c++) { + if (c == aggMe->block_y_index) + temp->addChannel(channel); + else + temp->addChannel(nullptr); + } + + return *temp; + }); + } +}; + +} // namespace conv2d_memory_fusion + +#endif diff --git a/src/conv2d_memory_fusion/headers/ConvResultToChunks.h b/src/conv2d_memory_fusion/headers/ConvResultToChunks.h index 8beaf6eb..99a30bf0 100644 --- a/src/conv2d_memory_fusion/headers/ConvResultToChunks.h +++ b/src/conv2d_memory_fusion/headers/ConvResultToChunks.h @@ -1,112 +1,112 @@ -#ifndef CONV2D_CONV_RESULT_TO_CHUNKS_H -#define CONV2D_CONV_RESULT_TO_CHUNKS_H - -#include "MultiSelectionComp.h" - -#include "Lambda.h" -#include "LambdaCreationFunctions.h" - -#include "PDBString.h" -#include "PDBVector.h" - -#include "FFMatrixBlock.h" -#include "ImageChunk.h" - -#include - -using namespace pdb; - -namespace conv2d_memory_fusion { - -// template -class ConvResultToChunks - : public MultiSelectionComp { - -public: - ENABLE_DEEP_COPY - - int num_images; - - ConvResultToChunks() {} - - ConvResultToChunks(int num_images) : num_images(num_images) {} - - Lambda getSelection(Handle<::FFMatrixBlock> checkMe) override { - return makeLambda(checkMe, - [](Handle<::FFMatrixBlock> &checkMe) { return true; }); - } - - Lambda>> - getProjection(Handle<::FFMatrixBlock> checkMe) override { - return makeLambda(checkMe, [this](Handle<::FFMatrixBlock> &checkMe) { - Vector> result; - - assert(checkMe->getTotalRowNums() % num_images == 0); - int group_size = checkMe->getTotalRowNums() / num_images; - std::cout << "[ConvResultToChunks] BLOCK ID: " - << checkMe->getBlockRowIndex() << ", " - << checkMe->getBlockColIndex() << std::endl; - - int origin_x = checkMe->getBlockRowIndex() * checkMe->getRowNums(); - int origin_y = checkMe->getBlockColIndex() * checkMe->getColNums(); - - std::cout << "[ConvResultToChunks] ORIGIN: " << origin_x << ", " - << origin_y << std::endl; - std::cout << "[ConvResultToChunks] GROUP_SIZE: " << group_size - << std::endl; - - // Strip away the padding - int real_x = checkMe->isLastRowBlock() - ? checkMe->getTotalRowNums() % checkMe->getRowNums() - : checkMe->getRowNums(); - int real_y = checkMe->isLastColBlock() - ? checkMe->getTotalColNums() % checkMe->getColNums() - : checkMe->getColNums(); - - int image_id = floor(origin_x / group_size); - int done = origin_x % group_size; - int left = min(group_size - done, real_x); - int size = left * real_y; - - std::cout << "[ConvResultToChunks] image_id: " << image_id - << ", done: " << done << ", left: " << left - << ", size: " << size << std::endl; - - /* - size = left * real_y or if left > checkMe->getRowNums(), - checkMe->getRowNums() * real_y block_row = image_id y_index = - checkMe->getBlockColIndex() block_row_start = done - */ - - int i = 0; - Handle chunk = nullptr; - double *data = checkMe->getValue().rawData->c_ptr(); - - while (i < real_x) { - for (int j = 0; j < real_y; j++) { - chunk = makeObject( - left, 1, done, - checkMe->getBlockColIndex() * checkMe->getColNums() + j, image_id); - for (int pos = 0; pos < left; pos++) { - chunk->getChunk().push_back( - data[(i + pos) * checkMe->getColNums() + j]); - } - result.push_back(chunk); - } - - i += left; - left = min(real_x - i, group_size); - image_id++; - done = 0; - size = left * real_y; - } - - return result; - }); - } -}; - -} // namespace conv2d_memory_fusion - -#endif +#ifndef CONV2D_CONV_RESULT_TO_CHUNKS_H +#define CONV2D_CONV_RESULT_TO_CHUNKS_H + +#include "MultiSelectionComp.h" + +#include "Lambda.h" +#include "LambdaCreationFunctions.h" + +#include "PDBString.h" +#include "PDBVector.h" + +#include "FFMatrixBlock.h" +#include "ImageChunk.h" + +#include + +using namespace pdb; + +namespace conv2d_memory_fusion { + +// template +class ConvResultToChunks + : public MultiSelectionComp { + +public: + ENABLE_DEEP_COPY + + int num_images; + + ConvResultToChunks() {} + + ConvResultToChunks(int num_images) : num_images(num_images) {} + + Lambda getSelection(Handle<::FFMatrixBlock> checkMe) override { + return makeLambda(checkMe, + [](Handle<::FFMatrixBlock> &checkMe) { return true; }); + } + + Lambda>> + getProjection(Handle<::FFMatrixBlock> checkMe) override { + return makeLambda(checkMe, [this](Handle<::FFMatrixBlock> &checkMe) { + Vector> result; + + assert(checkMe->getTotalRowNums() % num_images == 0); + int group_size = checkMe->getTotalRowNums() / num_images; + std::cout << "[ConvResultToChunks] BLOCK ID: " + << checkMe->getBlockRowIndex() << ", " + << checkMe->getBlockColIndex() << std::endl; + + int origin_x = checkMe->getBlockRowIndex() * checkMe->getRowNums(); + int origin_y = checkMe->getBlockColIndex() * checkMe->getColNums(); + + std::cout << "[ConvResultToChunks] ORIGIN: " << origin_x << ", " + << origin_y << std::endl; + std::cout << "[ConvResultToChunks] GROUP_SIZE: " << group_size + << std::endl; + + // Strip away the padding + int real_x = checkMe->isLastRowBlock() + ? checkMe->getTotalRowNums() % checkMe->getRowNums() + : checkMe->getRowNums(); + int real_y = checkMe->isLastColBlock() + ? checkMe->getTotalColNums() % checkMe->getColNums() + : checkMe->getColNums(); + + int image_id = floor(origin_x / group_size); + int done = origin_x % group_size; + int left = min(group_size - done, real_x); + int size = left * real_y; + + std::cout << "[ConvResultToChunks] image_id: " << image_id + << ", done: " << done << ", left: " << left + << ", size: " << size << std::endl; + + /* + size = left * real_y or if left > checkMe->getRowNums(), + checkMe->getRowNums() * real_y block_row = image_id y_index = + checkMe->getBlockColIndex() block_row_start = done + */ + + int i = 0; + Handle chunk = nullptr; + float *data = checkMe->getValue().rawData->c_ptr(); + + while (i < real_x) { + for (int j = 0; j < real_y; j++) { + chunk = makeObject( + left, 1, done, + checkMe->getBlockColIndex() * checkMe->getColNums() + j, image_id); + for (int pos = 0; pos < left; pos++) { + chunk->getChunk().push_back( + data[(i + pos) * checkMe->getColNums() + j]); + } + result.push_back(chunk); + } + + i += left; + left = min(real_x - i, group_size); + image_id++; + done = 0; + size = left * real_y; + } + + return result; + }); + } +}; + +} // namespace conv2d_memory_fusion + +#endif diff --git a/src/conv2d_memory_fusion/headers/Image.h b/src/conv2d_memory_fusion/headers/Image.h index fb472749..cef3b699 100644 --- a/src/conv2d_memory_fusion/headers/Image.h +++ b/src/conv2d_memory_fusion/headers/Image.h @@ -1,94 +1,94 @@ -#ifndef CONV2D_IMAGE_H -#define CONV2D_IMAGE_H - -#include "Matrix3D.h" -#include "PDBObject.h" - -#include - -namespace conv2d_memory_fusion { -class Image : public Matrix3D { -public: - int index; - - ENABLE_DEEP_COPY - Image() {} - Image(int index, int x, int y, int z) : index(index), Matrix3D(x, y, z) {} - - int &getKey() { return index; } - - Image &getValue() { return *this; } - - int get_x(int padding) { return x + padding * 2; } - - int get_y(int padding) { return y + padding * 2; } - - int get_h_slides(int padding, int kernel_size, int stride) { - return floor((get_y(padding) - kernel_size) / stride) + 1; - } - - int get_v_slides(int padding, int kernel_size, int stride) { - return floor((get_x(padding) - kernel_size) / stride) + 1; - } - - // TODO: odd Padding, odd strides, odd kernels - // Maybe another class that can save state and operate on this img? - double get_win_value(int window, int i, int j, int stride, int channel, - int kernel_size, int padding) { - int h_slides = get_h_slides(padding, kernel_size, stride); - int v_slides = get_v_slides(padding, kernel_size, stride); - - // calculate window origin - int r = floor(window / h_slides); - int c = window % h_slides; - - // Find where i,j lies in the kernel window - // Also adjust for padding - // We assume that padding is out of bounds of the actual array - int x_index = ((r * stride) + i) - padding; - int y_index = ((c * stride) + j) - padding; - - if (padding > 0 && - (x_index < 0 || y_index < 0 || x_index >= x || y_index >= y)) - return 0; - - int flat_index = x_index * y + y_index; - return matrices[channel]->getValue().rawData->c_ptr()[flat_index]; - } - - int get_conv2d_matrix_rows(int kernel_size, int stride, int padding) { - int v_slides = get_v_slides(padding, kernel_size, stride); - int h_slides = get_h_slides(padding, kernel_size, stride); - - return v_slides * h_slides; - } - - int get_conv2d_matrix_cols(int kernel_size, int stride, int padding) { - return z * kernel_size * kernel_size; - } - - int get_conv2d_window_count(int kernel_size, int stride, int padding) { - int h_slides = get_h_slides(padding, kernel_size, stride); - int v_slides = get_v_slides(padding, kernel_size, stride); - std::cout << "[IMAGE] h_slides: " << h_slides << ", v_slides: " << v_slides << std::endl; - std::cout << "[IMAGE] kernel: " << kernel_size << ", strides: " << stride << ", padding: " << padding << std::endl; - - return h_slides * v_slides; - } - - int get_num_channels() { return z; } - - Image &operator+(Image &addMeIn) { - for (int c = 0; c < z; c++) { - // assert((getMatrixAtIndex(c) == nullptr && addMeIn.getMatrixAtIndex(c) != nullptr) || (getMatrixAtIndex(c) != nullptr && addMeIn.getMatrixAtIndex(c) == nullptr)); - if (getMatrixAtIndex(c) == nullptr) { - setMatrixAtIndex(c, addMeIn.getMatrixAtIndex(c)); - } - } - - return *this; - } -}; -} // namespace conv2d_memory_fusion - -#endif +#ifndef CONV2D_IMAGE_H +#define CONV2D_IMAGE_H + +#include "Matrix3D.h" +#include "PDBObject.h" + +#include + +namespace conv2d_memory_fusion { +class Image : public Matrix3D { +public: + int index; + + ENABLE_DEEP_COPY + Image() {} + Image(int index, int x, int y, int z) : index(index), Matrix3D(x, y, z) {} + + int &getKey() { return index; } + + Image &getValue() { return *this; } + + int get_x(int padding) { return x + padding * 2; } + + int get_y(int padding) { return y + padding * 2; } + + int get_h_slides(int padding, int kernel_size, int stride) { + return floor((get_y(padding) - kernel_size) / stride) + 1; + } + + int get_v_slides(int padding, int kernel_size, int stride) { + return floor((get_x(padding) - kernel_size) / stride) + 1; + } + + // TODO: odd Padding, odd strides, odd kernels + // Maybe another class that can save state and operate on this img? + float get_win_value(int window, int i, int j, int stride, int channel, + int kernel_size, int padding) { + int h_slides = get_h_slides(padding, kernel_size, stride); + int v_slides = get_v_slides(padding, kernel_size, stride); + + // calculate window origin + int r = floor(window / h_slides); + int c = window % h_slides; + + // Find where i,j lies in the kernel window + // Also adjust for padding + // We assume that padding is out of bounds of the actual array + int x_index = ((r * stride) + i) - padding; + int y_index = ((c * stride) + j) - padding; + + if (padding > 0 && + (x_index < 0 || y_index < 0 || x_index >= x || y_index >= y)) + return 0; + + int flat_index = x_index * y + y_index; + return matrices[channel]->getValue().rawData->c_ptr()[flat_index]; + } + + int get_conv2d_matrix_rows(int kernel_size, int stride, int padding) { + int v_slides = get_v_slides(padding, kernel_size, stride); + int h_slides = get_h_slides(padding, kernel_size, stride); + + return v_slides * h_slides; + } + + int get_conv2d_matrix_cols(int kernel_size, int stride, int padding) { + return z * kernel_size * kernel_size; + } + + int get_conv2d_window_count(int kernel_size, int stride, int padding) { + int h_slides = get_h_slides(padding, kernel_size, stride); + int v_slides = get_v_slides(padding, kernel_size, stride); + std::cout << "[IMAGE] h_slides: " << h_slides << ", v_slides: " << v_slides << std::endl; + std::cout << "[IMAGE] kernel: " << kernel_size << ", strides: " << stride << ", padding: " << padding << std::endl; + + return h_slides * v_slides; + } + + int get_num_channels() { return z; } + + Image &operator+(Image &addMeIn) { + for (int c = 0; c < z; c++) { + // assert((getMatrixAtIndex(c) == nullptr && addMeIn.getMatrixAtIndex(c) != nullptr) || (getMatrixAtIndex(c) != nullptr && addMeIn.getMatrixAtIndex(c) == nullptr)); + if (getMatrixAtIndex(c) == nullptr) { + setMatrixAtIndex(c, addMeIn.getMatrixAtIndex(c)); + } + } + + return *this; + } +}; +} // namespace conv2d_memory_fusion + +#endif diff --git a/src/conv2d_memory_fusion/headers/ImageBlock.h b/src/conv2d_memory_fusion/headers/ImageBlock.h index 80e8eef1..0b3ea749 100644 --- a/src/conv2d_memory_fusion/headers/ImageBlock.h +++ b/src/conv2d_memory_fusion/headers/ImageBlock.h @@ -1,95 +1,95 @@ - -#ifndef CONV2D_IMAGE_BLOCK_H -#define CONV2D_IMAGE_BLOCK_H - -#include - -#include "Handle.h" -#include "Object.h" -#include "PDBMap.h" -#include "PDBString.h" -#include "PDBVector.h" - -#include "ImageChunk.h" - -using namespace pdb; - -namespace conv2d_memory_fusion { - -class ImageBlock : public pdb::Object { - -public: - int block_x_index; - int block_x_size; - int block_y_size; - int block_y_index; - Map> chunks; - long key; - - int feature_count; - - ENABLE_DEEP_COPY - - // Default constructor: - ImageBlock() {} - - // Default destructor: - ~ImageBlock() {} - - // Constructor with arguments: - ImageBlock(ImageChunk &chunk, int block_x_size) - : block_y_size(chunk.y_size), block_y_index(chunk.y_index), - block_x_size(block_x_size), - block_x_index((int)(chunk.getChunkActualRowIndex() / block_x_size)) { - feature_count = chunk.getSize(); - chunks[chunk.getChunkActualRowIndex()] = chunk.getChunk(); - key = block_x_index * 10000 + block_y_index; - } - - Map> &getBlock() { return chunks; } - - long &getKey() { return key; } - - ImageBlock &getValue() { return *this; } - - int getActualBlockSize() { return chunks.size(); } - - void dump() { - Map> &rhs = getBlock(); - std::cout << "ImageBlock: " << block_x_index << ", " << block_y_index << std::endl; - std::cout << "block size: " << block_x_size << ", " << block_y_size << ", key: " << key << std::endl; - auto iter = rhs.begin(); - while (iter != rhs.end()) { - int myKey = (*iter).key; - std::cout << "Key: " << myKey << " -> "; - Vector &data = (*iter).value; - for (int i = 0; i < data.size(); i++) - std::cout << data[i] << ", "; - std::cout << std::endl; - ++iter; - } - } - - ImageBlock &operator+(ImageBlock &addMeIn) { - - Map> &rhs = addMeIn.getBlock(); - auto iter = rhs.begin(); - while (iter != rhs.end()) { - int myKey = (*iter).key; - if (chunks.count(myKey) == 0) { - chunks[myKey] = (*iter).value; - //std::cout << "[ImageBlock] Merging bucket " << key << " adding key " - //<< myKey << std::endl; - } else { - //std::cout << "[ImageBlock] Merging bucket " << key << " NOT adding key " - //<< myKey << std::endl; - } - ++iter; - } - return *this; - } -}; - -} // namespace conv2d_memory_fusion - -#endif + +#ifndef CONV2D_IMAGE_BLOCK_H +#define CONV2D_IMAGE_BLOCK_H + +#include + +#include "Handle.h" +#include "Object.h" +#include "PDBMap.h" +#include "PDBString.h" +#include "PDBVector.h" + +#include "ImageChunk.h" + +using namespace pdb; + +namespace conv2d_memory_fusion { + +class ImageBlock : public pdb::Object { + +public: + int block_x_index; + int block_x_size; + int block_y_size; + int block_y_index; + Map> chunks; + long key; + + int feature_count; + + ENABLE_DEEP_COPY + + // Default constructor: + ImageBlock() {} + + // Default destructor: + ~ImageBlock() {} + + // Constructor with arguments: + ImageBlock(ImageChunk &chunk, int block_x_size) + : block_y_size(chunk.y_size), block_y_index(chunk.y_index), + block_x_size(block_x_size), + block_x_index((int)(chunk.getChunkActualRowIndex() / block_x_size)) { + feature_count = chunk.getSize(); + chunks[chunk.getChunkActualRowIndex()] = chunk.getChunk(); + key = block_x_index * 10000 + block_y_index; + } + + Map> &getBlock() { return chunks; } + + long &getKey() { return key; } + + ImageBlock &getValue() { return *this; } + + int getActualBlockSize() { return chunks.size(); } + + void dump() { + Map> &rhs = getBlock(); + std::cout << "ImageBlock: " << block_x_index << ", " << block_y_index << std::endl; + std::cout << "block size: " << block_x_size << ", " << block_y_size << ", key: " << key << std::endl; + auto iter = rhs.begin(); + while (iter != rhs.end()) { + int myKey = (*iter).key; + std::cout << "Key: " << myKey << " -> "; + Vector &data = (*iter).value; + for (int i = 0; i < data.size(); i++) + std::cout << data[i] << ", "; + std::cout << std::endl; + ++iter; + } + } + + ImageBlock &operator+(ImageBlock &addMeIn) { + + Map> &rhs = addMeIn.getBlock(); + auto iter = rhs.begin(); + while (iter != rhs.end()) { + int myKey = (*iter).key; + if (chunks.count(myKey) == 0) { + chunks[myKey] = (*iter).value; + //std::cout << "[ImageBlock] Merging bucket " << key << " adding key " + //<< myKey << std::endl; + } else { + //std::cout << "[ImageBlock] Merging bucket " << key << " NOT adding key " + //<< myKey << std::endl; + } + ++iter; + } + return *this; + } +}; + +} // namespace conv2d_memory_fusion + +#endif diff --git a/src/conv2d_memory_fusion/headers/ImageBlockToMatrix.h b/src/conv2d_memory_fusion/headers/ImageBlockToMatrix.h index d7ceffc7..ceb58f92 100644 --- a/src/conv2d_memory_fusion/headers/ImageBlockToMatrix.h +++ b/src/conv2d_memory_fusion/headers/ImageBlockToMatrix.h @@ -1,81 +1,81 @@ -#ifndef CONV2D_IMAGE_BLOCK_TO_MATRIX_H -#define CONV2D_IMAGE_BLOCK_TO_MATRIX_H - -#include -#include - -#include "SelectionComp.h" - -#include "Lambda.h" -#include "LambdaCreationFunctions.h" - -#include "PDBString.h" -#include "PDBVector.h" - -#include "FFMatrixBlock.h" -#include "ImageBlock.h" - -using namespace pdb; - -namespace conv2d_memory_fusion { - -class ImageBlockToMatrix : public SelectionComp<::FFMatrixBlock, ImageBlock> { - -public: - int block_x; - int block_y; - bool padding; - int batch_size; - int total_features; - - ENABLE_DEEP_COPY - - ImageBlockToMatrix() {} - - ImageBlockToMatrix(int block_x, int block_y, bool padding, int batch_size, - int total_features) - : block_x(block_x), block_y(block_y), padding(padding), - batch_size(batch_size), total_features(total_features) {} - - Lambda getSelection(Handle checkMe) override { - return makeLambda(checkMe, - [](Handle &checkMe) { return true; }); - } - - Lambda> - getProjection(Handle checkMe) override { - return makeLambda(checkMe, [this](Handle &checkMe) { - int real_block_x = padding ? block_x : checkMe->getActualBlockSize(); - int real_block_y = - padding - ? block_y - : min(block_y, total_features - checkMe->block_y_index * block_y); - - Handle<::FFMatrixBlock> myData = makeObject<::FFMatrixBlock>( - checkMe->block_x_index, checkMe->block_y_index, real_block_x, - real_block_y, batch_size, total_features, true); - - Map> &chunk = checkMe->getBlock(); - // Because each chunk has a mapping between the actual index of the - // comment and the features. These may be stored out of order in the - // map. - int offset = checkMe->block_x_index * checkMe->block_x_size; - - for (int x = 0; x < real_block_x; x++) { - for (int y = 0; y < real_block_y; y++) { - double data = - x >= checkMe->getActualBlockSize() || y >= checkMe->feature_count - ? 0 - : chunk[offset + x][y]; - (*(myData->getRawDataHandle()))[x * real_block_y + y] = data; - } - } - - return myData; - }); - } -}; - -} // namespace conv2d_memory_fusion - -#endif +#ifndef CONV2D_IMAGE_BLOCK_TO_MATRIX_H +#define CONV2D_IMAGE_BLOCK_TO_MATRIX_H + +#include +#include + +#include "SelectionComp.h" + +#include "Lambda.h" +#include "LambdaCreationFunctions.h" + +#include "PDBString.h" +#include "PDBVector.h" + +#include "FFMatrixBlock.h" +#include "ImageBlock.h" + +using namespace pdb; + +namespace conv2d_memory_fusion { + +class ImageBlockToMatrix : public SelectionComp<::FFMatrixBlock, ImageBlock> { + +public: + int block_x; + int block_y; + bool padding; + int batch_size; + int total_features; + + ENABLE_DEEP_COPY + + ImageBlockToMatrix() {} + + ImageBlockToMatrix(int block_x, int block_y, bool padding, int batch_size, + int total_features) + : block_x(block_x), block_y(block_y), padding(padding), + batch_size(batch_size), total_features(total_features) {} + + Lambda getSelection(Handle checkMe) override { + return makeLambda(checkMe, + [](Handle &checkMe) { return true; }); + } + + Lambda> + getProjection(Handle checkMe) override { + return makeLambda(checkMe, [this](Handle &checkMe) { + int real_block_x = padding ? block_x : checkMe->getActualBlockSize(); + int real_block_y = + padding + ? block_y + : min(block_y, total_features - checkMe->block_y_index * block_y); + + Handle<::FFMatrixBlock> myData = makeObject<::FFMatrixBlock>( + checkMe->block_x_index, checkMe->block_y_index, real_block_x, + real_block_y, batch_size, total_features, true); + + Map> &chunk = checkMe->getBlock(); + // Because each chunk has a mapping between the actual index of the + // comment and the features. These may be stored out of order in the + // map. + int offset = checkMe->block_x_index * checkMe->block_x_size; + + for (int x = 0; x < real_block_x; x++) { + for (int y = 0; y < real_block_y; y++) { + float data = + x >= checkMe->getActualBlockSize() || y >= checkMe->feature_count + ? 0 + : chunk[offset + x][y]; + (*(myData->getRawDataHandle()))[x * real_block_y + y] = data; + } + } + + return myData; + }); + } +}; + +} // namespace conv2d_memory_fusion + +#endif diff --git a/src/conv2d_memory_fusion/headers/ImageChunk.h b/src/conv2d_memory_fusion/headers/ImageChunk.h index 468f1b8a..f41146bd 100644 --- a/src/conv2d_memory_fusion/headers/ImageChunk.h +++ b/src/conv2d_memory_fusion/headers/ImageChunk.h @@ -1,44 +1,44 @@ -#ifndef CONV2D_IMAGE_CHUNK_H -#define CONV2D_IMAGE_CHUNK_H - -#include "Handle.h" -#include "PDBObject.h" -#include "PDBVector.h" - -namespace conv2d_memory_fusion { -class ImageChunk : public pdb::Object { -public: - int block_row; - int y_index; - pdb::Vector chunk; - int chunk_size; - int y_size; - - int block_row_start; - - ENABLE_DEEP_COPY - ImageChunk() {} - - ImageChunk(int size, int y_size, int block_row, int y_index, int block_row_start) - : chunk_size(size), y_size(y_size), block_row(block_row), - y_index(y_index), block_row_start(block_row_start) { - chunk.resize(size); - } - - int getChunkActualRowIndex() { return block_row_start + block_row; } - - pdb::Vector &getChunk() { return chunk; } - - void dump() { - std::cout << "size: " << chunk_size << ", block_row: " << block_row << ", y_index: " << y_index << ", block_row_start" << block_row_start << std::endl; - for (int i = 0; i < chunk.size(); i++) { - std::cout << chunk[i] << ","; - } - std::cout << std::endl; - } - - int getSize() { return chunk.size(); } -}; -} // namespace conv2d_memory_fusion - +#ifndef CONV2D_IMAGE_CHUNK_H +#define CONV2D_IMAGE_CHUNK_H + +#include "Handle.h" +#include "PDBObject.h" +#include "PDBVector.h" + +namespace conv2d_memory_fusion { +class ImageChunk : public pdb::Object { +public: + int block_row; + int y_index; + pdb::Vector chunk; + int chunk_size; + int y_size; + + int block_row_start; + + ENABLE_DEEP_COPY + ImageChunk() {} + + ImageChunk(int size, int y_size, int block_row, int y_index, int block_row_start) + : chunk_size(size), y_size(y_size), block_row(block_row), + y_index(y_index), block_row_start(block_row_start) { + chunk.resize(size); + } + + int getChunkActualRowIndex() { return block_row_start + block_row; } + + pdb::Vector &getChunk() { return chunk; } + + void dump() { + std::cout << "size: " << chunk_size << ", block_row: " << block_row << ", y_index: " << y_index << ", block_row_start" << block_row_start << std::endl; + for (int i = 0; i < chunk.size(); i++) { + std::cout << chunk[i] << ","; + } + std::cout << std::endl; + } + + int getSize() { return chunk.size(); } +}; +} // namespace conv2d_memory_fusion + #endif \ No newline at end of file diff --git a/src/conv2d_memory_fusion/headers/KernelBiasJoin.h b/src/conv2d_memory_fusion/headers/KernelBiasJoin.h index 8383eebe..45a1d7dc 100644 --- a/src/conv2d_memory_fusion/headers/KernelBiasJoin.h +++ b/src/conv2d_memory_fusion/headers/KernelBiasJoin.h @@ -1,55 +1,55 @@ -#ifndef CONV2D_KERNEL_BIAS_JOIN_H -#define CONV2D_KERNEL_BIAS_JOIN_H - -#include "FFMatrixBlock.h" -#include "JoinComp.h" -#include "Lambda.h" -#include "LambdaCreationFunctions.h" - -#include - -using namespace pdb; - -namespace conv2d_memory_fusion { -class KernelBiasJoin - : public JoinComp<::FFMatrixBlock, ::FFMatrixBlock, ::FFMatrixBlock> { - -public: - ENABLE_DEEP_COPY - - KernelBiasJoin() {} - - Lambda getSelection(Handle<::FFMatrixBlock> in1, - Handle<::FFMatrixBlock> in2) override { - return makeLambdaFromMethod(in1, getBlockRowIndex) == - makeLambdaFromMethod(in2, getBlockRowIndex) && - makeLambdaFromMethod(in1, isLastColBlock); - } - - Lambda> - getProjection(Handle<::FFMatrixBlock> in1, - Handle<::FFMatrixBlock> in2) override { - return makeLambda( - in1, in2, - [](Handle<::FFMatrixBlock> &in1, Handle<::FFMatrixBlock> &in2) { - // in1 is kernel, in2 is bias - assert(in2->getColNums() == 1 && in2->getTotalColNums() == 1 && - in1->getRowNums() == in2->getRowNums() && - in2->isLastColBlock()); - - double *in1Data = in1->getValue().rawData->c_ptr(); - double *in2Data = in2->getValue().rawData->c_ptr(); - int pos = in1->getTotalColNums() % in1->getColNums() - 1; - - // Copy bias values - for (int i = 0; i < in1->getRowNums(); i++) { - in1Data[i * in1->getColNums() + pos] = in2Data[i]; - } - - return in1; - }); - } -}; -} // namespace conv2d_memory_fusion - -#endif +#ifndef CONV2D_KERNEL_BIAS_JOIN_H +#define CONV2D_KERNEL_BIAS_JOIN_H + +#include "FFMatrixBlock.h" +#include "JoinComp.h" +#include "Lambda.h" +#include "LambdaCreationFunctions.h" + +#include + +using namespace pdb; + +namespace conv2d_memory_fusion { +class KernelBiasJoin + : public JoinComp<::FFMatrixBlock, ::FFMatrixBlock, ::FFMatrixBlock> { + +public: + ENABLE_DEEP_COPY + + KernelBiasJoin() {} + + Lambda getSelection(Handle<::FFMatrixBlock> in1, + Handle<::FFMatrixBlock> in2) override { + return makeLambdaFromMethod(in1, getBlockRowIndex) == + makeLambdaFromMethod(in2, getBlockRowIndex) && + makeLambdaFromMethod(in1, isLastColBlock); + } + + Lambda> + getProjection(Handle<::FFMatrixBlock> in1, + Handle<::FFMatrixBlock> in2) override { + return makeLambda( + in1, in2, + [](Handle<::FFMatrixBlock> &in1, Handle<::FFMatrixBlock> &in2) { + // in1 is kernel, in2 is bias + assert(in2->getColNums() == 1 && in2->getTotalColNums() == 1 && + in1->getRowNums() == in2->getRowNums() && + in2->isLastColBlock()); + + float *in1Data = in1->getValue().rawData->c_ptr(); + float *in2Data = in2->getValue().rawData->c_ptr(); + int pos = in1->getTotalColNums() % in1->getColNums() - 1; + + // Copy bias values + for (int i = 0; i < in1->getRowNums(); i++) { + in1Data[i * in1->getColNums() + pos] = in2Data[i]; + } + + return in1; + }); + } +}; +} // namespace conv2d_memory_fusion + +#endif diff --git a/src/conv2d_memory_fusion/headers/KernelToChunks.h b/src/conv2d_memory_fusion/headers/KernelToChunks.h index bab9cc57..b0f0921c 100644 --- a/src/conv2d_memory_fusion/headers/KernelToChunks.h +++ b/src/conv2d_memory_fusion/headers/KernelToChunks.h @@ -1,89 +1,89 @@ -#ifndef CONV2D_KERNEL_TO_CHUNKS_H -#define CONV2D_KERNEL_TO_CHUNKS_H - -#include "MultiSelectionComp.h" - -#include "Lambda.h" -#include "LambdaCreationFunctions.h" - -#include "PDBString.h" -#include "PDBVector.h" - -#include "FFMatrixBlock.h" -#include "ImageChunk.h" -#include "Kernel.h" - -#include - -using namespace pdb; - -namespace conv2d_memory_fusion { - -// template -class KernelToChunks : public MultiSelectionComp { - -public: - ENABLE_DEEP_COPY - - int block_y; - - KernelToChunks() {} - KernelToChunks(int block_y) : block_y(block_y) {} - - Lambda getSelection(Handle checkMe) override { - return makeLambda(checkMe, [](Handle &checkMe) { return true; }); - } - - Lambda>> - getProjection(Handle checkMe) override { - return makeLambda(checkMe, [this](Handle &checkMe) { - Vector> result; - - int y_index = 0; - - int row_width = checkMe->x * checkMe->y * checkMe->z; - // Kernel flattens to a single row - // The position of this row is the index of the kernel - int block_row = 0; - int block_row_start = checkMe->getKey(); - - Handle chunk = - makeObject(block_y, block_y, block_row, y_index, block_row_start); - - for (int c = 0; c < checkMe->get_num_channels(); c++) { - double *data = - checkMe->getMatrixAtIndex(c)->getValue().rawData->c_ptr(); - - for (int i = 0; i < checkMe->x * checkMe->y; i++) { - if (chunk->getSize() > 0 && chunk->getSize() % block_y == 0) { - result.push_back(chunk); - y_index++; - chunk = makeObject(block_y, block_y, block_row, y_index, - block_row_start); - } - - chunk->getChunk().push_back(data[i]); - } - } - - // Add the last chunk of last row - // Set placeholder for bias - if (chunk->getSize() % block_y == 0) { - result.push_back(chunk); - y_index++; - chunk = - makeObject(block_y, block_y, block_row_start, y_index, block_row_start); - } - - chunk->getChunk().push_back(0); - result.push_back(chunk); - - return result; - }); - } -}; - -} // namespace conv2d_memory_fusion - -#endif +#ifndef CONV2D_KERNEL_TO_CHUNKS_H +#define CONV2D_KERNEL_TO_CHUNKS_H + +#include "MultiSelectionComp.h" + +#include "Lambda.h" +#include "LambdaCreationFunctions.h" + +#include "PDBString.h" +#include "PDBVector.h" + +#include "FFMatrixBlock.h" +#include "ImageChunk.h" +#include "Kernel.h" + +#include + +using namespace pdb; + +namespace conv2d_memory_fusion { + +// template +class KernelToChunks : public MultiSelectionComp { + +public: + ENABLE_DEEP_COPY + + int block_y; + + KernelToChunks() {} + KernelToChunks(int block_y) : block_y(block_y) {} + + Lambda getSelection(Handle checkMe) override { + return makeLambda(checkMe, [](Handle &checkMe) { return true; }); + } + + Lambda>> + getProjection(Handle checkMe) override { + return makeLambda(checkMe, [this](Handle &checkMe) { + Vector> result; + + int y_index = 0; + + int row_width = checkMe->x * checkMe->y * checkMe->z; + // Kernel flattens to a single row + // The position of this row is the index of the kernel + int block_row = 0; + int block_row_start = checkMe->getKey(); + + Handle chunk = + makeObject(block_y, block_y, block_row, y_index, block_row_start); + + for (int c = 0; c < checkMe->get_num_channels(); c++) { + float *data = + checkMe->getMatrixAtIndex(c)->getValue().rawData->c_ptr(); + + for (int i = 0; i < checkMe->x * checkMe->y; i++) { + if (chunk->getSize() > 0 && chunk->getSize() % block_y == 0) { + result.push_back(chunk); + y_index++; + chunk = makeObject(block_y, block_y, block_row, y_index, + block_row_start); + } + + chunk->getChunk().push_back(data[i]); + } + } + + // Add the last chunk of last row + // Set placeholder for bias + if (chunk->getSize() % block_y == 0) { + result.push_back(chunk); + y_index++; + chunk = + makeObject(block_y, block_y, block_row_start, y_index, block_row_start); + } + + chunk->getChunk().push_back(0); + result.push_back(chunk); + + return result; + }); + } +}; + +} // namespace conv2d_memory_fusion + +#endif diff --git a/src/conv2d_memory_fusion/headers/PartialMatrix.h b/src/conv2d_memory_fusion/headers/PartialMatrix.h index 3bb78846..4d970744 100644 --- a/src/conv2d_memory_fusion/headers/PartialMatrix.h +++ b/src/conv2d_memory_fusion/headers/PartialMatrix.h @@ -1,49 +1,49 @@ -#ifndef CONV2D_PARTIAL_MATRIX_H -#define CONV2D_PARTIAL_MATRIX_H - -#include "Handle.h" -#include "PDBObject.h" -#include "PDBVector.h" - -namespace conv2d_memory_fusion { -class PartialMatrix : public pdb::Object { -public: - int origin_x; - int origin_y; - int x; - int y; - int key; - - pdb::Vector partial; - ENABLE_DEEP_COPY - - PartialMatrix() {} - - PartialMatrix(int origin_x, int origin_y, int x, int y, int key) - : origin_x(origin_x), origin_y(origin_y), x(x), y(y), key(key) { - partial.resize(x * y); - } - - // Copy x*y elements from offset - void copy_partial(pdb::Vector &from, int offset) { - for (int i = 0; i < x * y; i++) { - partial[i] = from[offset + i]; - } - } - - pdb::Vector &getPartial() { return partial; } - - void dump() { - std::cout << "x: " << x << ", y: " << y << ", origin_x: " << origin_x - << ", origin_y: " << origin_y << ", key: " << key << std::endl; - for (int i = 0; i < partial.size(); i++) { - std::cout << partial[i] << ","; - } - std::cout << std::endl; - } - - int getSize() { return partial.size(); } -}; -} // namespace conv2d_memory_fusion - +#ifndef CONV2D_PARTIAL_MATRIX_H +#define CONV2D_PARTIAL_MATRIX_H + +#include "Handle.h" +#include "PDBObject.h" +#include "PDBVector.h" + +namespace conv2d_memory_fusion { +class PartialMatrix : public pdb::Object { +public: + int origin_x; + int origin_y; + int x; + int y; + int key; + + pdb::Vector partial; + ENABLE_DEEP_COPY + + PartialMatrix() {} + + PartialMatrix(int origin_x, int origin_y, int x, int y, int key) + : origin_x(origin_x), origin_y(origin_y), x(x), y(y), key(key) { + partial.resize(x * y); + } + + // Copy x*y elements from offset + void copy_partial(pdb::Vector &from, int offset) { + for (int i = 0; i < x * y; i++) { + partial[i] = from[offset + i]; + } + } + + pdb::Vector &getPartial() { return partial; } + + void dump() { + std::cout << "x: " << x << ", y: " << y << ", origin_x: " << origin_x + << ", origin_y: " << origin_y << ", key: " << key << std::endl; + for (int i = 0; i < partial.size(); i++) { + std::cout << partial[i] << ","; + } + std::cout << std::endl; + } + + int getSize() { return partial.size(); } +}; +} // namespace conv2d_memory_fusion + #endif \ No newline at end of file diff --git a/src/conv2d_memory_fusion/source/ImageUtils.cc b/src/conv2d_memory_fusion/source/ImageUtils.cc index 5df49ad9..6ac4f712 100644 --- a/src/conv2d_memory_fusion/source/ImageUtils.cc +++ b/src/conv2d_memory_fusion/source/ImageUtils.cc @@ -139,7 +139,7 @@ void load_imgs_from_file(PDBClient &pdbClient, string path, String dbName, void loadRandomImages(int width, int height, int channels, int numOfImages, pdb::PDBClient &pdbClient, pdb::String dbName, pdb::String setName, int pageSize) { std::string errMsg; - pdb::makeObjectAllocatorBlock(20 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(1024 * 1024 * 1024, true); pdb::Handle>> storeImages = pdb::makeObject>>(); @@ -148,8 +148,8 @@ void loadRandomImages(int width, int height, int channels, int numOfImages, pdb: std::mt19937 e2(rd()); - std::uniform_real_distribution<> distp(0.0001, 0.5); - std::uniform_real_distribution<> distn(-0.5, -0.0001); + std::uniform_real_distribution distp(0.0001, 0.5); + std::uniform_real_distribution distn(-0.5, -0.0001); auto gen = std::bind(std::uniform_int_distribution<>(0, 1), std::default_random_engine()); @@ -163,7 +163,7 @@ void loadRandomImages(int width, int height, int channels, int numOfImages, pdb: for (int i = 0; i < width; i++) { for (int j = 0; j < height; j++) { - double data = (bool)gen() ? distn(e2) : distp(e2); + float data = (bool)gen() ? distn(e2) : distp(e2); (*(channel->getRawDataHandle()))[i * height + j] = data; } } @@ -180,7 +180,7 @@ void loadRandomImages(int width, int height, int channels, int numOfImages, pdb: } imageCount--; - pdb::makeObjectAllocatorBlock(20 * 1024 * 1024, true); + pdb::makeObjectAllocatorBlock(1024 * 1024 * 1024, true); storeImages = pdb::makeObject>>(); } } diff --git a/src/conv2d_proj/headers/Conv2DSelect.h b/src/conv2d_proj/headers/Conv2DSelect.h index 4acdd8f1..c90f3388 100644 --- a/src/conv2d_proj/headers/Conv2DSelect.h +++ b/src/conv2d_proj/headers/Conv2DSelect.h @@ -1,235 +1,235 @@ -#ifndef CONV2DSELECT_H -#define CONV2DSELECT_H - -#include "SelectionComp.h" -#include "Lambda.h" -#include "LambdaCreationFunctions.h" -#include "TensorData.h" - -//LA libraries: -#include -#include - -//ATen libraries: -#include -#include - -using namespace pdb; - -//stride = 1 - -class Conv2DSelect: public SelectionComp{ - -public: - - ENABLE_DEEP_COPY - - Conv2DSelect() {} - - Conv2DSelect(Handle filters, std::string convMode = "aten-conv2d", int stride = 1) { - - //make sure it's a 3D tensor - assert(filters->numRanks = 4); - - //set up the kernel and kernel dimensions - this->kernel = filters; - - //kernel count - this->nk = (*(filters->dimensions))[0]; - - //C - this->zk = (*(filters->dimensions))[1]; - - //H - this->yk = (*(filters->dimensions))[2]; - - //W - this->xk = (*(filters->dimensions))[3]; - - //set up the mode of the convolutional operation - this->conv2dMode = convMode; - - this->stride = stride; - - } - - Lambda getSelection(Handle checkMe) override { - - return makeLambda(checkMe, [](Handle& checkMe) { return true; }); - - } - - - Handle runEigenSpatial(TensorData& input, int z, int y, int x, int stride) { - - Eigen::TensorMap> a (input.rawData->c_ptr(), z, y, x); - - //Eigen::Tensor c = a.convolve(b1) - - Eigen::TensorMap> b1(kernel->rawData->c_ptr(), nk, zk, yk, xk); - - - //NumDims of input: 3 - //kernelFilters: nk - //kernelChannels: zk - //kernelRows: yk - //kernelCols: xk - //kernelRowsEff = kernelRows + (kernelRows-1)*(row_in_stride-1) = yk - //kernelColsEff = kernelCols + (kernelCols-1)*(col_in_stride-1) = xk - - - //contract_dims - Eigen::array, 1> contract_dims; - contract_dims[0] = Eigen::IndexPair(1, 0); - - - //out_height = (inputRows - kernelRowsEff) / stride + 1 = (y - yk) / stride + 1 - //out_width = (inputCols - kernelColsEff) / stride + 1 = (x - xk) / stride + 1 - - int oy = calculateOutputDimension(y, yk, stride); - int ox = calculateOutputDimension(x, xk, stride); - - //pre_contract_dims - Eigen::array pre_contract_dims; - pre_contract_dims[0] = zk * yk * xk; - pre_contract_dims[1] = (oy) * (ox); - - //post_contract_dims - Eigen::array post_contract_dims; - post_contract_dims[0] = nk; - post_contract_dims[1] = (oy); - post_contract_dims[2] = (ox); - - //kernel dims - Eigen::array kernel_dims; - kernel_dims[0] = nk; - kernel_dims[1] = zk * yk * xk; - - - //create the output - - Handle> dimensions = makeObject>(3); - - dimensions->push_back(nk); - - dimensions->push_back(oy); - - dimensions->push_back(ox); - - Handle out = makeObject(3, dimensions); - - float * mempool = (float *) malloc (nk * oy * ox * sizeof(float)); - - Eigen::TensorMap> c (mempool, nk, oy, ox); - - c = b1.reshape(kernel_dims) - .contract( - a.extract_image_patches(yk, xk, stride, stride, 1, 1, - Eigen::PADDING_VALID) - .reshape(pre_contract_dims), contract_dims) - .reshape(post_contract_dims); - - - /* - c = a.extract_image_patches(yk, xk, 1, 1, 1, 1, Eigen::PADDING_VALID) - - .reshape(Eigen::array({(y - yk + 1 ) * (x - xk + 1 ), zk * yk * xk})) - .contract(b1.reshape(Eigen::array({yk * xk, nk})), contract_dims) - .reshape(Eigen::array({ (x - xk + 1 ), (y - yk + 1 ), nk })); - - */ - - memcpy (out->rawData->c_ptr(), mempool, nk * oy * ox * sizeof(float)); - - return out; - } - - Handle runAtenConv2d(TensorData& input, int z, int y, int x, int stride) { - - //input data - at::Tensor a = at::from_blob(input.rawData->c_ptr(), {1, z, y, x}); - - at::Tensor b = at::from_blob(kernel->rawData->c_ptr(), {nk, zk, yk, xk}); - - // bias length = kernel count = nk - at::Tensor bias = at::zeros({nk}, at::kInt); - //perform the convolutional operation - auto c = at::conv2d(a, b, bias, stride); - - //create the output - int oy = calculateOutputDimension(y, yk, stride); - int ox = calculateOutputDimension(x, xk, stride); - Handle> dimensions = makeObject>(3); - - dimensions->push_back(nk); - - dimensions->push_back(oy); - - dimensions->push_back(ox); - - Handle out = makeObject(3, dimensions); - memcpy(out->rawData->c_ptr(), c.storage().data(), nk * (oy) * (ox) * sizeof(float)); - - return out; - } - - - - Lambda> getProjection(Handle checkMe) override { - - return makeLambda(checkMe, [&](Handle& checkMe) { - - - TensorData input = *checkMe; - - assert (input.numRanks = 3); - - //set up input dimensions - - //C - int z = (*(input.dimensions))[0]; - - //H - int y = (*(input.dimensions))[1]; - - //W - int x = (*(input.dimensions))[2]; - - - if (conv2dMode == "eigen-spatial") { - return runEigenSpatial(input, z, y, x, stride); - } else { - return runAtenConv2d(input, z, y, x, stride); - } - }); - } - -private: - - //multiple 2D-filters, each filter has many channels - Handle kernel = nullptr; - - //conv2d-mode: - //--"eigen-spatial": https://github.com/tensorflow/tensorflow/blob/v1.13.1/tensorflow/core/kernels/eigen_spatial_convolutions.h#L1688 - //https://github.com/pytorch/pytorch/blob/master/caffe2/operators/conv_op_eigen.cc - //--"aten-conv2d": https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/native/Convolution.cpp - String conv2dMode = "aten-conv2d"; - - - int nk; - - int xk; - - int yk; - - int zk; - - unsigned int stride; - - static int calculateOutputDimension(int inputDimention, int filterDimention, int stride) { - return (inputDimention - filterDimention) / stride + 1; - } - -}; - -#endif +#ifndef CONV2DSELECT_H +#define CONV2DSELECT_H + +#include "SelectionComp.h" +#include "Lambda.h" +#include "LambdaCreationFunctions.h" +#include "TensorData.h" + +//LA libraries: +#include +#include + +//ATen libraries: +#include +#include + +using namespace pdb; + +//stride = 1 + +class Conv2DSelect: public SelectionComp{ + +public: + + ENABLE_DEEP_COPY + + Conv2DSelect() {} + + Conv2DSelect(Handle filters, std::string convMode = "aten-conv2d", int stride = 1) { + + //make sure it's a 3D tensor + assert(filters->numRanks = 4); + + //set up the kernel and kernel dimensions + this->kernel = filters; + + //kernel count + this->nk = (*(filters->dimensions))[0]; + + //C + this->zk = (*(filters->dimensions))[1]; + + //H + this->yk = (*(filters->dimensions))[2]; + + //W + this->xk = (*(filters->dimensions))[3]; + + //set up the mode of the convolutional operation + this->conv2dMode = convMode; + + this->stride = stride; + + } + + Lambda getSelection(Handle checkMe) override { + + return makeLambda(checkMe, [](Handle& checkMe) { return true; }); + + } + + + Handle runEigenSpatial(TensorData& input, int z, int y, int x, int stride) { + + Eigen::TensorMap> a (input.rawData->c_ptr(), z, y, x); + + //Eigen::Tensor c = a.convolve(b1) + + Eigen::TensorMap> b1(kernel->rawData->c_ptr(), nk, zk, yk, xk); + + + //NumDims of input: 3 + //kernelFilters: nk + //kernelChannels: zk + //kernelRows: yk + //kernelCols: xk + //kernelRowsEff = kernelRows + (kernelRows-1)*(row_in_stride-1) = yk + //kernelColsEff = kernelCols + (kernelCols-1)*(col_in_stride-1) = xk + + + //contract_dims + Eigen::array, 1> contract_dims; + contract_dims[0] = Eigen::IndexPair(1, 0); + + + //out_height = (inputRows - kernelRowsEff) / stride + 1 = (y - yk) / stride + 1 + //out_width = (inputCols - kernelColsEff) / stride + 1 = (x - xk) / stride + 1 + + int oy = calculateOutputDimension(y, yk, stride); + int ox = calculateOutputDimension(x, xk, stride); + + //pre_contract_dims + Eigen::array pre_contract_dims; + pre_contract_dims[0] = zk * yk * xk; + pre_contract_dims[1] = (oy) * (ox); + + //post_contract_dims + Eigen::array post_contract_dims; + post_contract_dims[0] = nk; + post_contract_dims[1] = (oy); + post_contract_dims[2] = (ox); + + //kernel dims + Eigen::array kernel_dims; + kernel_dims[0] = nk; + kernel_dims[1] = zk * yk * xk; + + + //create the output + + Handle> dimensions = makeObject>(3); + + dimensions->push_back(nk); + + dimensions->push_back(oy); + + dimensions->push_back(ox); + + Handle out = makeObject(3, dimensions); + + float * mempool = (float *) malloc (nk * oy * ox * sizeof(float)); + + Eigen::TensorMap> c (mempool, nk, oy, ox); + + c = b1.reshape(kernel_dims) + .contract( + a.extract_image_patches(yk, xk, stride, stride, 1, 1, + Eigen::PADDING_VALID) + .reshape(pre_contract_dims), contract_dims) + .reshape(post_contract_dims); + + + /* + c = a.extract_image_patches(yk, xk, 1, 1, 1, 1, Eigen::PADDING_VALID) + + .reshape(Eigen::array({(y - yk + 1 ) * (x - xk + 1 ), zk * yk * xk})) + .contract(b1.reshape(Eigen::array({yk * xk, nk})), contract_dims) + .reshape(Eigen::array({ (x - xk + 1 ), (y - yk + 1 ), nk })); + + */ + + memcpy (out->rawData->c_ptr(), mempool, nk * oy * ox * sizeof(float)); + + return out; + } + + Handle runAtenConv2d(TensorData& input, int z, int y, int x, int stride) { + + //input data + at::Tensor a = at::from_blob(input.rawData->c_ptr(), {1, z, y, x}); + + at::Tensor b = at::from_blob(kernel->rawData->c_ptr(), {nk, zk, yk, xk}); + + // bias length = kernel count = nk + at::Tensor bias = at::zeros({nk}, at::kFloat); + //perform the convolutional operation + auto c = at::conv2d(a, b, bias, stride); + + //create the output + int oy = calculateOutputDimension(y, yk, stride); + int ox = calculateOutputDimension(x, xk, stride); + Handle> dimensions = makeObject>(3); + + dimensions->push_back(nk); + + dimensions->push_back(oy); + + dimensions->push_back(ox); + + Handle out = makeObject(3, dimensions); + memcpy(out->rawData->c_ptr(), c.storage().data(), nk * (oy) * (ox) * sizeof(float)); + + return out; + } + + + + Lambda> getProjection(Handle checkMe) override { + + return makeLambda(checkMe, [&](Handle& checkMe) { + + + TensorData input = *checkMe; + + assert (input.numRanks = 3); + + //set up input dimensions + + //C + int z = (*(input.dimensions))[0]; + + //H + int y = (*(input.dimensions))[1]; + + //W + int x = (*(input.dimensions))[2]; + + + if (conv2dMode == "eigen-spatial") { + return runEigenSpatial(input, z, y, x, stride); + } else { + return runAtenConv2d(input, z, y, x, stride); + } + }); + } + +private: + + //multiple 2D-filters, each filter has many channels + Handle kernel = nullptr; + + //conv2d-mode: + //--"eigen-spatial": https://github.com/tensorflow/tensorflow/blob/v1.13.1/tensorflow/core/kernels/eigen_spatial_convolutions.h#L1688 + //https://github.com/pytorch/pytorch/blob/master/caffe2/operators/conv_op_eigen.cc + //--"aten-conv2d": https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/native/Convolution.cpp + String conv2dMode = "aten-conv2d"; + + + int nk; + + int xk; + + int yk; + + int zk; + + unsigned int stride; + + static int calculateOutputDimension(int inputDimention, int filterDimention, int stride) { + return (inputDimention - filterDimention) / stride + 1; + } + +}; + +#endif diff --git a/src/objectModel/headers/BuiltinPDBObjects.h b/src/objectModel/headers/BuiltinPDBObjects.h index f0034522..a6195115 100644 --- a/src/objectModel/headers/BuiltinPDBObjects.h +++ b/src/objectModel/headers/BuiltinPDBObjects.h @@ -1,138 +1,138 @@ // Auto-generated by code in SConstruct -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/SetScan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StoragePagePinned.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/KMeansDoubleVector.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveDatabase.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/AggregationJobStage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageRemoveDatabase.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ComputePlan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageTestSetCopy.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ExecuteQuery.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ResourceInfo.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/NodeInfo.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageCollectStatsResponse.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSharedPage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatCreateDatabaseRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetSetRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddObjectInLoop.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PDBObjectPrototype.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ZB_Company.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Array.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/AggregationMap.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DepartmentEmployeeAges.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageCleanup.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Employee.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BroadcastJoinBuildHTJobStage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Avg.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetDatabaseRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetTypeResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddData.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/OptimizedDepartmentEmployees.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ExecuteComputation.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BuiltinPartialResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DepartmentEmployees.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/HashPartitionedJoinBuildHTJobStage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageRemoveTempSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageCleanup.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveTempSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/JoinPairArray.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageNoMorePage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddTempSetResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageClearSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StoragePinPage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StringIntPair.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatPrintCatalogRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/JoinMap.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/GenericBlock.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DoubleSumResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageCollectStats.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CloseConnection.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueryPermitResponse.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSharedLibraryByNameRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DispatcherAddData.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddTempSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DeleteSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/OptimizedSupervisor.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSyncRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TopKQueue.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/OptimizedEmployee.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BaseQuery.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/SumResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/RequestResources.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatDeleteDatabaseRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/WriteUserSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageUnpinPage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/AvgResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Count.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DepartmentTotal.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatRegisterType.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DoubleVectorResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageGetData.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Holder.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StoragePinBytes.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetDatabaseResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ScanDoubleVectorSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageExportSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BackendTestSetCopy.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BackendTestSetScan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageGetSetPages.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddTempSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddSharedPage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageGetDataResponse.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/SetIdentifier.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatTypeNameSearchResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageExportSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Ack.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TupleSetExecuteQuery.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/MyEmployee.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddDatabase.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DoubleVector.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageBytesPinned.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatPrintCatalogResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageTestSetScan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageGetStats.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PairArray.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueryDone.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSyncResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ShutDown.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DispatcherRegisterPartitionPolicy.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageRemoveHashSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/SimpleRequestResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatDeleteSetRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueriesAndPlan.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatalogUserTypeMetadata.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveHashSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetType.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddObject.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSetWithPartition.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DistributedStorageClearSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatGetSetResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/LambdaIdentifier.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/DoneWithResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ListOfNodes.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ProcessorFactory.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSetObjectTypeRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddType.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageRemoveUserSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/KeepGoing.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PDBVector.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PDBMap.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/NodeDispatcherData.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Set.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/ScanUserSet.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueryOutput.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/PlaceOfQueryPlanner.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/BackendExecuteSelection.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Supervisor.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/Nothing.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatSharedLibraryResult.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TensorBlockMeta.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/StorageAddDatabase.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TupleSetJobStage.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/QueryPermit.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/GetListOfNodes.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/CatCreateSetRequest.h" -#include "/home/ubuntu/netsdb/src/builtInPDBObjects/headers/TensorBlockIdentifier.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Ack.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/AggregationJobStage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/AggregationMap.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Array.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Avg.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/AvgResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BackendExecuteSelection.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BackendTestSetCopy.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BackendTestSetScan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BaseQuery.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BroadcastJoinBuildHTJobStage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/BuiltinPartialResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatalogUserTypeMetadata.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatCreateDatabaseRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatCreateSetRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatDeleteDatabaseRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatDeleteSetRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetDatabaseRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetDatabaseResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetSetRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetSetResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetType.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatGetTypeResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatPrintCatalogRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatPrintCatalogResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatRegisterType.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSetObjectTypeRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSharedLibraryByNameRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSharedLibraryResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSyncRequest.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatSyncResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CatTypeNameSearchResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/CloseConnection.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ComputePlan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Count.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DeleteSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DepartmentEmployeeAges.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DepartmentEmployees.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DepartmentTotal.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DispatcherAddData.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DispatcherRegisterPartitionPolicy.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddDatabase.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSetWithPartition.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddSharedPage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageAddTempSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageCleanup.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageClearSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageExportSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveDatabase.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveHashSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DistributedStorageRemoveTempSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DoneWithResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DoubleSumResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DoubleVector.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/DoubleVectorResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Employee.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ExecuteComputation.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ExecuteQuery.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/GenericBlock.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/GetListOfNodes.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/HashPartitionedJoinBuildHTJobStage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Holder.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/JoinMap.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/JoinPairArray.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/KeepGoing.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/KMeansDoubleVector.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/LambdaIdentifier.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ListOfNodes.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/MyEmployee.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/NodeDispatcherData.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/NodeInfo.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Nothing.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/OptimizedDepartmentEmployees.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/OptimizedEmployee.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/OptimizedSupervisor.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PairArray.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PDBMap.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PDBObjectPrototype.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PDBVector.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/PlaceOfQueryPlanner.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ProcessorFactory.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueriesAndPlan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueryDone.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueryOutput.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueryPermit.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/QueryPermitResponse.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/RequestResources.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ResourceInfo.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ScanDoubleVectorSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ScanUserSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Set.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/SetIdentifier.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/SetScan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ShutDown.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/SimpleRequestResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddData.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddDatabase.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddObject.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddObjectInLoop.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddSharedPage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddTempSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddTempSetResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageAddType.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageBytesPinned.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageCleanup.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageClearSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageCollectStats.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageCollectStatsResponse.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageExportSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageGetData.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageGetDataResponse.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageGetSetPages.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageGetStats.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageNoMorePage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StoragePagePinned.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StoragePinBytes.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StoragePinPage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageRemoveDatabase.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageRemoveHashSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageRemoveTempSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageRemoveUserSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageTestSetCopy.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageTestSetScan.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StorageUnpinPage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/StringIntPair.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/SumResult.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/Supervisor.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TensorBlockIdentifier.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TensorBlockMeta.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TopKQueue.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TupleSetExecuteQuery.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/TupleSetJobStage.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/WriteUserSet.h" +#include "/home/pavan/netsdb/src/builtInPDBObjects/headers/ZB_Company.h" diff --git a/src/serverFunctionalities/source/FrontendQueryTestServer.cc b/src/serverFunctionalities/source/FrontendQueryTestServer.cc index 3a37f1c1..ca5c9a1a 100644 --- a/src/serverFunctionalities/source/FrontendQueryTestServer.cc +++ b/src/serverFunctionalities/source/FrontendQueryTestServer.cc @@ -705,7 +705,6 @@ void FrontendQueryTestServer::registerHandlers(PDBServer& forMe) { } } - getFunctionality().cleanup(false); Handle result = nullptr; if (needsRemoveCombinerSet == true) { @@ -725,7 +724,6 @@ void FrontendQueryTestServer::registerHandlers(PDBServer& forMe) { // now, we send back the result - getFunctionality().cleanup(false); if (result == nullptr) { result = makeObject(outDatabaseName, outSetName); result->setNumPages(outputSet->getNumPages()); diff --git a/src/serverFunctionalities/source/PangeaStorageServer.cc b/src/serverFunctionalities/source/PangeaStorageServer.cc index 6ef06bef..4775d6d1 100644 --- a/src/serverFunctionalities/source/PangeaStorageServer.cc +++ b/src/serverFunctionalities/source/PangeaStorageServer.cc @@ -166,8 +166,6 @@ void PangeaStorageServer::cleanup(bool flushOrNot) { writeBackRecords(a.first, flushOrNot); } std::cout << "Now there are " << totalObjects << " new objects stored in storage" << std::endl; - PDB_COUT << "sleep for 1 second to wait for all data gets flushed" << std::endl; - sleep(1); PDB_COUT << "cleaned up for storage..." << std::endl; } diff --git a/src/tests/source/Conv2dMemFuseTest.cc b/src/tests/source/Conv2dMemFuseTest.cc index 5ec12eb8..e4356c71 100644 --- a/src/tests/source/Conv2dMemFuseTest.cc +++ b/src/tests/source/Conv2dMemFuseTest.cc @@ -1,694 +1,788 @@ -#include -#include -#include -#include - -#include "FFMatrixBlock.h" -#include "FFMatrixData.h" -#include "FFMatrixMeta.h" - -#include "Image.h" -#include "ImageBlock.h" -#include "ImageBlockToMatrix.h" -#include "ImageChunk.h" -#include "ImageChunksToBlock.h" -#include "ImageToChunks.h" -#include "Matrix3D.h" - -#include "Kernel.h" -#include "KernelBiasJoin.h" -#include "KernelToChunks.h" -#include "ConvResultToChunks.h" -#include "ConvChunksToImage.h" - -#include "FFMatrixUtil.h" -#include "ImageUtils.h" - -#include "FFTransposeMult.h" -#include "FFAggMatrix.h" - -#include "PDBClient.h" -#include "PDBVector.h" -#include "ScanUserSet.h" -#include "WriteUserSet.h" - -using namespace std; - -namespace test_common { -void create_database(pdb::PDBClient &pdbClient, std::string dbName, - std::vector sets, bool whetherToCreateDatabase=true) { - std::string errMsg; - std::cout << "Setting up database..." << std::endl; - - if (whetherToCreateDatabase) { - pdbClient.removeDatabase(dbName, errMsg); - if (!pdbClient.createDatabase(dbName, errMsg)) { - cout << "Not able to create database " << dbName << ": " << errMsg << endl; - } else { - cout << "Created database " << dbName << endl; - } - } - for (auto &s : sets) { - pdbClient.removeSet(dbName, s, errMsg); - if (!pdbClient.createSet( - dbName, s, errMsg, (size_t)128 * (size_t)1024 * (size_t)1024, s)) { - cout << "Not able to create set " + s + ": " + errMsg; - } else { - cout << "Created set " << s << ".\n"; - } - } -} - -void conv2d_op(pdb::PDBClient &pdbClient, std::string dbName, std::string image, - std::string kernel, std::string res, bool materializeModel) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, image); - pdb::Handle readB = - makeObject>(dbName, kernel); - - pdb::Handle join = pdb::makeObject(); - join->setInput(0, readA); - join->setInput(1, readB); - - // make the aggregation - pdb::Handle myAggregation = pdb::makeObject(); - myAggregation->setInput(join); - - Handle myWriteSet = - makeObject>(dbName, res); - myWriteSet->setInput(myAggregation); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "conv2d", materializeModel, myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - //pdbClient.flushData(errMsg); -} - -void conv2d_result_to_chunks(pdb::PDBClient &pdbClient, int images, std::string dbName, std::string result, std::string to) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, result); - - pdb::Handle chonk = pdb::makeObject(images); - chonk->setInput(readA); - - Handle myWriteSet = - makeObject>(dbName, to); - myWriteSet->setInput(chonk); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "result-chonk", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - //pdbClient.flushData(errMsg); -} -} // namespace test_common - -namespace img_conv_flatten { -void verify_data(pdb::PDBClient &pdbClient, std::string dbName, - std::string setName) { - std::cout << "Verifying data..." << std::endl; - - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - auto it = pdbClient.getSetIterator(dbName, setName); - - for (auto r : it) { - for (int c = 0; c < r->get_num_channels(); c++) { - pdb::Handle chan = r->getMatrixAtIndex(c); - std::cout << "Channel " << c << std::endl; - double *data = chan->getValue().rawData->c_ptr(); - for (int i = 0; i < r->x * r->y; i++) { - std::cout << data[i] << ","; - } - std::cout << std::endl; - } - std::cout << std::endl; - } -} - -void verify_chunks(pdb::PDBClient &pdbClient, std::string dbName, - std::string setName) { - std::cout << "Verifying chunks..." << std::endl; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - auto it = pdbClient.getSetIterator(dbName, setName); - - for (auto r : it) { - r->dump(); - } -} - -void verify_blocks(pdb::PDBClient &pdbClient, std::string dbName, - std::string setName) { - std::cout << "Verifying blocks..." << std::endl; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - auto it = pdbClient.getSetIterator(dbName, setName); - - for (auto r : it) { - r->dump(); - } -} - -void verify_matrices(pdb::PDBClient &pdbClient, std::string dbName, - std::string setName) { - std::cout << "Verifying matrices..." << std::endl; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - auto it = pdbClient.getSetIterator(dbName, setName); - - for (auto r : it) { - std::cout << "Block " << r->getBlockRowIndex() << "," - << r->getBlockColIndex() << std::endl; - double *data = r->getValue().rawData->c_ptr(); - for (int i = 0; i < r->getColNums() * r->getRowNums(); i++) { - std::cout << data[i] << ","; - } - std::cout << std::endl; - } -} - -void img_to_chunks(pdb::PDBClient &pdbClient, std::string dbName, - std::string from, std::string to, int block_x, int block_y, - int strides, int kernel, int padding) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, from); - - pdb::Handle chunk = - pdb::makeObject(block_y, kernel, strides, padding); - chunk->setInput(readA); - - Handle myWriteSet = - makeObject>(dbName, to); - myWriteSet->setInput(chunk); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "chunk", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - //pdbClient.flushData(errMsg); -} - -void chunks_to_blocks(pdb::PDBClient &pdbClient, std::string dbName, - std::string from, std::string to, int block_x) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, from); - - pdb::Handle block = - pdb::makeObject(block_x); - block->setInput(readA); - - Handle myWriteSet = - makeObject>(dbName, to); - myWriteSet->setInput(block); - - // run the computation - if (!pdbClient.executeComputations(errMsg, from, myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - //pdbClient.flushData(errMsg); -} - -void blocks_to_matrix(pdb::PDBClient &pdbClient, std::string dbName, - std::string from, std::string to, int block_x, - int block_y, bool block_padding, int X, int Y) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, from); - - pdb::Handle matrix = - pdb::makeObject(block_x, block_y, - block_padding, X, Y); - matrix->setInput(readA); - - Handle myWriteSet = - makeObject>(dbName, to); - myWriteSet->setInput(matrix); - - // run the computation - if (!pdbClient.executeComputations(errMsg, from, myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - //pdbClient.flushData(errMsg); -} -} // namespace img_conv_flatten - -namespace kernel_conv_flatten { -void kernel_to_chunks(pdb::PDBClient &pdbClient, std::string dbName, - std::string from, std::string to, int block_x, - int block_y) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, from); - - pdb::Handle chunk = - pdb::makeObject(block_y); - chunk->setInput(readA); - - Handle myWriteSet = - makeObject>(dbName, to); - myWriteSet->setInput(chunk); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "kernel-chunk", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - //pdbClient.flushData(errMsg); -} - -void kernel_bias_join(pdb::PDBClient &pdbClient, std::string dbName, - std::string a, std::string b, std::string to) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, a); - pdb::Handle readB = - makeObject>(dbName, b); - - pdb::Handle join = - pdb::makeObject(); - join->setInput(0, readA); - join->setInput(1, readB); - - Handle myWriteSet = - makeObject>(dbName, to); - myWriteSet->setInput(join); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "kernel_bias_join", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - //pdbClient.flushData(errMsg); -} -} // namespace kernel_conv_flatten - - -void test_conv2d_multiply(pdb::PDBClient &pdbClient, std::string dbName, - std::string imageset, std::string kernelset, - std::string biasset, bool reloadData, bool materializeModel) { - std::vector allSets{imageset, kernelset, biasset, "temp_kernel", "temp_kernel1", "temp_kernel2", "kernel_flat", "temp_image", "temp_image1", "temp_image2", - "result", "result_chunked", "result_chunked1", "result_chunked2"}; - - std::vector intermediateSets{ "temp_kernel", "temp_kernel1", "temp_kernel2", "kernel_flat", - "temp_image", "temp_image1", "temp_image2", - "result", "result_chunked", "result_chunked1", "result_chunked2"}; - std::vector intermediateSets1{"temp_image", "temp_image1", "temp_image2", - "result", "result_chunked", "result_chunked1", "result_chunked2"}; - std::string errMsg; - - int block_x = 32; - int block_y = 32; - int kernel = 1;//kernel shape should be $kernelx$kernel (e.g., 7x7) - int strides = 1; - int padding = 0; - bool block_padding = true; - - int height = 7, width = 7, channels = 512, numOfImages = 100; - int kHeight = 1, kWidth = 1, kChannels = 512, numOfFilters = 2048; - if (reloadData) { - - test_common::create_database(pdbClient, dbName, allSets, reloadData); - - // Load Image - std::cout << "Loading images....." << std::endl; - conv2d_memory_fusion::loadRandomImages(width, height, channels, numOfImages, pdbClient, dbName, imageset, 20); - std::cout << "Image loaded successfully....." << std::endl; - - #ifdef PROFILING_CONV2D - img_conv_flatten::verify_data(pdbClient, dbName, imageset); - #endif - - // Load Kernel - std::cout << "Loading kernel....." << std::endl; - conv2d_memory_fusion::loadRandomImages(kHeight, kWidth, kChannels, numOfFilters, pdbClient, dbName, kernelset, 20); - std::cout << "Kernel loaded successfully......" << std::endl; - #ifdef PROFILING_CONV2D - img_conv_flatten::verify_data(pdbClient, dbName, kernelset); - #endif - std::cout << "Loading bias data..." << std::endl; - ff::load_matrix_data(pdbClient, "/home/pavan/netsdb/bias_2048.np", dbName, biasset, - block_x, 1, !block_padding, !block_padding, errMsg); - std::cout<< "Successfully loaded matrix data bias-------------------" << std::endl; - #ifdef PROFILING_CONV2D - ff::print(pdbClient, dbName, biasset); - #endif - } else { - if (!materializeModel) - test_common::create_database(pdbClient, dbName, intermediateSets, reloadData); - else - test_common::create_database(pdbClient, dbName, intermediateSets1, reloadData); - } - - // Image ops - std::cout << "Running image ops..." << std::endl; - - auto image_begin = std::chrono::high_resolution_clock::now(); - img_conv_flatten::img_to_chunks(pdbClient, dbName, imageset, "temp_image", - block_x, block_y, strides, kWidth, padding); - - - #ifdef PROFILING_CONV2D - img_conv_flatten::verify_chunks(pdbClient, dbName, "temp_image"); - #endif - - img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp_image", - "temp_image1", block_x); - - #ifdef PROFILING_CONV2D - img_conv_flatten::verify_blocks(pdbClient, dbName, "temp_image1"); - #endif - - //X: num_rows per flattened image (112-7+1)(112-7+1) - //Y: num_cols per flattened image (7*7*3+1) why +1? - img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp_image1", - "temp_image2", block_x, block_y, - block_padding, (width - kWidth) * (height - kHeight), kWidth * kHeight * kChannels + 1); - - - //#ifdef PROFILING_CONV2D - ff::print(pdbClient, dbName, "temp_image2"); - //#endif - auto image_end = std::chrono::high_resolution_clock::now(); - - auto kernel_flat_begin = std::chrono::high_resolution_clock::now(); - - if (!materializeModel) { - // kernel ops - std::cout << "Running kernel ops..." << std::endl; - - - kernel_conv_flatten::kernel_to_chunks(pdbClient, dbName, kernelset, - "temp_kernel", block_x, block_y); - - #ifdef PROFILING_CONV2D - img_conv_flatten::verify_chunks(pdbClient, dbName, "temp_kernel"); - #endif - - img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp_kernel", - "temp_kernel1", block_x); - - #ifdef PROFILING_CONV2D - img_conv_flatten::verify_blocks(pdbClient, dbName, "temp_kernel1"); - #endif - - img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp_kernel1", - "temp_kernel2", block_x, block_y, - block_padding, 64, kWidth * kHeight * kChannels + 1); - - #ifdef PROFILING_CONV2D - img_conv_flatten::verify_matrices(pdbClient, dbName, "temp_kernel2"); - #endif - - // kernel bias join - std::cout << "Running kernel bias join..." << std::endl; - - - kernel_conv_flatten::kernel_bias_join(pdbClient, dbName, "temp_kernel2", - biasset, "kernel_flat"); - - #ifdef PROFILING_CONV2D - ff::print(pdbClient, dbName, "kernel_flat"); - #endif - } - auto kernel_flat_end = std::chrono::high_resolution_clock::now(); - - - // multiply - std::cout << "Running conv2d op..." << std::endl; - - auto conv2d_begin = std::chrono::high_resolution_clock::now(); - - test_common::conv2d_op(pdbClient, dbName, "temp_image2", "kernel_flat", "result", materializeModel); - - auto conv2d_end = std::chrono::high_resolution_clock::now(); - - #ifdef PROFILING_CONV2D - ff::print(pdbClient, dbName, "result"); - #endif - - - -/* test_common::conv2d_result_to_chunks(pdbClient, 100, dbName, "result", "result_chunked"); - -#ifdef PROFILING_CONV2D - img_conv_flatten::verify_chunks(pdbClient, dbName, "result_chunked"); -#endif - - img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "result_chunked", - "result_chunked1", 11236); - -#ifdef PROFILING_CONV2D - img_conv_flatten::verify_blocks(pdbClient, dbName, "result_chunked1"); -#endif - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, "result_chunked1"); - - pdb::Handle chonk = pdb::makeObject(106, 106, 64); - chonk->setInput(readA); - - Handle myWriteSet = - makeObject>(dbName, "result_chunked2"); - myWriteSet->setInput(chonk); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "result-img", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - } - -#ifdef PROFILING_CONV2D - pdbClient.flushData(errMsg); - - img_conv_flatten::verify_data(pdbClient, dbName, "result_chunked2"); -#endif - - auto result_end = std::chrono::high_resolution_clock::now(); -*/ - - - std::cout << "Time Duration for image ops: " - << std::chrono::duration_cast>(image_end - image_begin).count() - << " secs." << std::endl; - - std::cout << "Time Duration for kernel flatterning ops: " - << std::chrono::duration_cast>(kernel_flat_end - kernel_flat_begin).count() - << " secs." << std::endl; - - std::cout << "Time Duration for conv2d: " - << std::chrono::duration_cast>(conv2d_end - conv2d_begin).count() - << " secs." << std::endl; - - //std::cout << "Time Duration for result gathering: " - // << std::chrono::duration_cast>(result_end - result_begin).count() - // << " secs." << std::endl; - -} - -int main(int argc, char *argv[]) { - - string masterIp = "localhost"; - pdb::PDBLoggerPtr clientLogger = make_shared("Conv2DclientLog"); - pdb::PDBClient pdbClient(8108, masterIp, clientLogger, false, true); - pdb::CatalogClient catalogClient(8108, masterIp, clientLogger); - - string errMsg; - - bool reloadData = true; - if (argc > 1) { - if (strcmp(argv[1], "N")==0) { - reloadData = false; - } - } - - bool materializeModel = false; - if (argc > 2) { - if (strcmp(argv[2], "Y")==0) { - materializeModel = true; - } - } - - - // Load Libraries - - if (reloadData) { - - if (!pdbClient.registerType("libraries/libFFMatrixData.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libFFMatrixData.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libFFMatrixMeta.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libFFMatrixMeta.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libFFMatrixBlock.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libFFMatrixBlock.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseMatrix3D.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseMatrix3D.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseImage.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseImage.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseImageChunk.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseImageChunk.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseImageToChunks.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseImageToChunks.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseImageBlock.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseImageBlock.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseImageChunksToBlock.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseImageChunksToBlock.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseImageBlockToMatrix.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseImageBlockToMatrix.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseKernel.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseKernel.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseKernelToChunks.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseKernelToChunks.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseKernelBiasJoin.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseKernelBiasJoin.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libFFTransposeMult.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libFFTransposeMult.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libFFAggMatrix.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libFFAggMatrix.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseConvResultToChunks.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseConvResultToChunks.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseConvChunksToImage.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseConvChunksToImage.so" - << ": " << errMsg << endl; - exit(-1); - } - } - - string dbName = "conv2d"; - string img_set = "img"; - string kernel_set = "kernel"; - test_conv2d_multiply(pdbClient, dbName, img_set, kernel_set, "bias", reloadData, materializeModel); - - return 0; -} +#include +#include +#include +#include + +#include "FFMatrixBlock.h" +#include "FFMatrixData.h" +#include "FFMatrixMeta.h" + +#include "Image.h" +#include "ImageBlock.h" +#include "ImageBlockToMatrix.h" +#include "ImageChunk.h" +#include "ImageChunksToBlock.h" +#include "ImageToChunks.h" +#include "Matrix3D.h" + +#include "Kernel.h" +#include "KernelBiasJoin.h" +#include "KernelToChunks.h" +#include "ConvResultToChunks.h" +#include "ConvChunksToImage.h" + +#include "FFMatrixUtil.h" +#include "ImageUtils.h" + +#include "FFTransposeMult.h" +#include "FFAggMatrix.h" + +#include "PDBClient.h" +#include "PDBVector.h" +#include "ScanUserSet.h" +#include "WriteUserSet.h" + +using namespace std; + +namespace test_common { +void create_database(pdb::PDBClient &pdbClient, std::string dbName, + std::vector sets, bool whetherToCreateDatabase=true) { + std::string errMsg; + std::cout << "Setting up database..." << std::endl; + + if (whetherToCreateDatabase) { + pdbClient.removeDatabase(dbName, errMsg); + if (!pdbClient.createDatabase(dbName, errMsg)) { + cout << "Not able to create database " << dbName << ": " << errMsg << endl; + } else { + cout << "Created database " << dbName << endl; + } + } + for (auto &s : sets) { + pdbClient.removeSet(dbName, s, errMsg); + if (!pdbClient.createSet( + dbName, s, errMsg, (size_t)1800 * (size_t)1024 * (size_t)1024, s)) { + cout << "Not able to create set " + s + ": " + errMsg; + } else { + cout << "Created set " << s << ".\n"; + } + } +} + +void conv2d_op(pdb::PDBClient &pdbClient, std::string dbName, std::string image, + std::string kernel, std::string res, bool materializeModel) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 2047}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, image); + pdb::Handle readB = + makeObject>(dbName, kernel); + + pdb::Handle join = pdb::makeObject(); + join->setInput(0, readA); + join->setInput(1, readB); + + // make the aggregation + pdb::Handle myAggregation = pdb::makeObject(); + myAggregation->setInput(join); + + Handle myWriteSet = + makeObject>(dbName, res); + myWriteSet->setInput(myAggregation); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "conv2d", materializeModel, myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + //pdbClient.flushData(errMsg); +} + +void conv2d_result_to_chunks(pdb::PDBClient &pdbClient, int images, std::string dbName, std::string result, std::string to) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 2047}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, result); + + pdb::Handle chonk = pdb::makeObject(images); + chonk->setInput(readA); + + Handle myWriteSet = + makeObject>(dbName, to); + myWriteSet->setInput(chonk); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "result-chonk", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + //pdbClient.flushData(errMsg); +} +} // namespace test_common + +namespace img_conv_flatten { +void verify_data(pdb::PDBClient &pdbClient, std::string dbName, + std::string setName) { + std::cout << "Verifying data..." << std::endl; + + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 2047}; + + auto it = pdbClient.getSetIterator(dbName, setName); + + for (auto r : it) { + for (int c = 0; c < r->get_num_channels(); c++) { + pdb::Handle chan = r->getMatrixAtIndex(c); + std::cout << "Channel " << c << std::endl; + float *data = chan->getValue().rawData->c_ptr(); + for (int i = 0; i < r->x * r->y; i++) { + std::cout << data[i] << ","; + } + std::cout << std::endl; + } + std::cout << std::endl; + } +} + +void verify_chunks(pdb::PDBClient &pdbClient, std::string dbName, + std::string setName) { + std::cout << "Verifying chunks..." << std::endl; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 2047}; + + auto it = pdbClient.getSetIterator(dbName, setName); + + for (auto r : it) { + r->dump(); + } +} + +void verify_blocks(pdb::PDBClient &pdbClient, std::string dbName, + std::string setName) { + std::cout << "Verifying blocks..." << std::endl; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 2047}; + + auto it = pdbClient.getSetIterator(dbName, setName); + + for (auto r : it) { + r->dump(); + } +} + +void verify_matrices(pdb::PDBClient &pdbClient, std::string dbName, + std::string setName) { + std::cout << "Verifying matrices..." << std::endl; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 2047}; + + auto it = pdbClient.getSetIterator(dbName, setName); + + for (auto r : it) { + std::cout << "Block " << r->getBlockRowIndex() << "," + << r->getBlockColIndex() << std::endl; + float *data = r->getValue().rawData->c_ptr(); + for (int i = 0; i < r->getColNums() * r->getRowNums(); i++) { + std::cout << data[i] << ","; + } + std::cout << std::endl; + } +} + +void img_to_chunks(pdb::PDBClient &pdbClient, std::string dbName, + std::string from, std::string to, int block_x, int block_y, + int strides, int kernel, int padding) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 2047}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, from); + + pdb::Handle chunk = + pdb::makeObject(block_y, kernel, strides, padding); + chunk->setInput(readA); + + Handle myWriteSet = + makeObject>(dbName, to); + myWriteSet->setInput(chunk); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "chunk", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + //pdbClient.flushData(errMsg); +} + +void chunks_to_blocks(pdb::PDBClient &pdbClient, std::string dbName, + std::string from, std::string to, int block_x) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 2047}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, from); + + pdb::Handle block = + pdb::makeObject(block_x); + block->setInput(readA); + + Handle myWriteSet = + makeObject>(dbName, to); + myWriteSet->setInput(block); + + // run the computation + if (!pdbClient.executeComputations(errMsg, from, myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + //pdbClient.flushData(errMsg); +} + +void blocks_to_matrix(pdb::PDBClient &pdbClient, std::string dbName, + std::string from, std::string to, int block_x, + int block_y, bool block_padding, int X, int Y) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 2047}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, from); + + pdb::Handle matrix = + pdb::makeObject(block_x, block_y, + block_padding, X, Y); + matrix->setInput(readA); + + Handle myWriteSet = + makeObject>(dbName, to); + myWriteSet->setInput(matrix); + + // run the computation + if (!pdbClient.executeComputations(errMsg, from, myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + //pdbClient.flushData(errMsg); +} +} // namespace img_conv_flatten + +namespace kernel_conv_flatten { +void kernel_to_chunks(pdb::PDBClient &pdbClient, std::string dbName, + std::string from, std::string to, int block_x, + int block_y) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 2047}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, from); + + pdb::Handle chunk = + pdb::makeObject(block_y); + chunk->setInput(readA); + + Handle myWriteSet = + makeObject>(dbName, to); + myWriteSet->setInput(chunk); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "kernel-chunk", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + //pdbClient.flushData(errMsg); +} + +void kernel_bias_join(pdb::PDBClient &pdbClient, std::string dbName, + std::string a, std::string b, std::string to) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 2047}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, a); + pdb::Handle readB = + makeObject>(dbName, b); + + pdb::Handle join = + pdb::makeObject(); + join->setInput(0, readA); + join->setInput(1, readB); + + Handle myWriteSet = + makeObject>(dbName, to); + myWriteSet->setInput(join); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "kernel_bias_join", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + //pdbClient.flushData(errMsg); +} +} // namespace kernel_conv_flatten + + +void test_conv2d_multiply(pdb::PDBClient &pdbClient, std::string dbName, + std::string imageset, std::string kernelset, + std::string biasset, bool reloadData, bool materializeModel, + vector imageDims, vector kernelDims) { + std::vector allSets{imageset, kernelset, biasset, "temp_kernel", "temp_kernel1", "temp_kernel2", "kernel_flat", "temp_image", "temp_image1", "temp_image2", + "result", "result_chunked", "result_chunked1", "result_chunked2"}; + + std::vector intermediateSets{ "temp_kernel", "temp_kernel1", "temp_kernel2", "kernel_flat", + "temp_image", "temp_image1", "temp_image2", + "result", "result_chunked", "result_chunked1", "result_chunked2"}; + std::vector intermediateSets1{"temp_image", "temp_image1", "temp_image2", + "result", "result_chunked", "result_chunked1", "result_chunked2"}; + std::string errMsg; + + int block_x = 32; + int block_y = 32; + int kernel = 1;//kernel shape should be $kernelx$kernel (e.g., 7x7) + int strides = 1; + int padding = 0; + bool block_padding = true; + + int n, c, w, h; + int nk, ck, wk, hk; + + n = std::stoi(imageDims.at(0)); + c = std::stoi(imageDims.at(1)); + w = std::stoi(imageDims.at(2)); + h = std::stoi(imageDims.at(3)); + + nk = std::stoi(kernelDims.at(0)); + ck = std::stoi(kernelDims.at(1)); + wk = std::stoi(kernelDims.at(2)); + hk = std::stoi(kernelDims.at(3)); + + int height = h, width = w, channels = c, numOfImages = n; + int kHeight = hk, kWidth = wk, kChannels = ck, numOfFilters = nk; + + std::chrono::high_resolution_clock::time_point loadRandomImages_time; + std::chrono::high_resolution_clock::time_point loadRandomImages_end; + if (reloadData) { + + test_common::create_database(pdbClient, dbName, allSets, reloadData); + + // Load Image + std::cout << "Loading images....." << std::endl; + loadRandomImages_time = std::chrono::high_resolution_clock::now(); + conv2d_memory_fusion::loadRandomImages(width, height, channels, numOfImages, pdbClient, dbName, imageset, 20); + std::cout << "Image loaded successfully....." << std::endl; + loadRandomImages_end = std::chrono::high_resolution_clock::now(); + + #ifdef PROFILING_CONV2D + img_conv_flatten::verify_data(pdbClient, dbName, imageset); + #endif + + // Load Kernel + std::cout << "Loading kernel....." << std::endl; + conv2d_memory_fusion::loadRandomImages(kHeight, kWidth, kChannels, numOfFilters, pdbClient, dbName, kernelset, 20); + std::cout << "Kernel loaded successfully......" << std::endl; + #ifdef PROFILING_CONV2D + img_conv_flatten::verify_data(pdbClient, dbName, kernelset); + #endif + // std::cout << "Loading bias data..." << std::endl; + // ff::load_matrix_data(pdbClient, "/home/pavan/netsdb/bias_2048.np", dbName, biasset, + // block_x, 1, !block_padding, !block_padding, errMsg); + // std::cout<< "Successfully loaded matrix data bias-------------------" << std::endl; + #ifdef PROFILING_CONV2D + ff::print(pdbClient, dbName, biasset); + #endif + } else { + if (!materializeModel) + test_common::create_database(pdbClient, dbName, intermediateSets, reloadData); + else + test_common::create_database(pdbClient, dbName, intermediateSets1, reloadData); + } + + // Image ops + std::cout << "Running image ops..." << std::endl; + + auto image_begin = std::chrono::high_resolution_clock::now(); + img_conv_flatten::img_to_chunks(pdbClient, dbName, imageset, "temp_image", + block_x, block_y, strides, kWidth, padding); + + auto img_to_chunks_time = std::chrono::high_resolution_clock::now(); + + + #ifdef PROFILING_CONV2D + img_conv_flatten::verify_chunks(pdbClient, dbName, "temp_image"); + #endif + + auto verify_chunks_time_end = std::chrono::high_resolution_clock::now(); + + img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp_image", + "temp_image1", block_x); + + auto chunks_to_blocks_end = std::chrono::high_resolution_clock::now(); + + #ifdef PROFILING_CONV2D + img_conv_flatten::verify_blocks(pdbClient, dbName, "temp_image1"); + #endif + + auto verify_blocks_end = std::chrono::high_resolution_clock::now(); + + + //X: num_rows per flattened image (112-7+1)(112-7+1) + //Y: num_cols per flattened image (7*7*3+1) why +1? + img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp_image1", + "temp_image2", block_x, block_y, + block_padding, numOfImages * ((width + 2 * padding - kWidth) / strides + 1) * ((height + 2 * - kHeight) / strides + 1), kWidth * kHeight * kChannels + 1); + + + auto blocks_to_matrix_end = std::chrono::high_resolution_clock::now(); + + + //#ifdef PROFILING_CONV2D + ff::print(pdbClient, dbName, "temp_image2"); + //#endif + auto image_end = std::chrono::high_resolution_clock::now(); + + auto kernel_flat_begin = std::chrono::high_resolution_clock::now(); + + if (!materializeModel) { + // kernel ops + std::cout << "Running kernel ops..." << std::endl; + + + kernel_conv_flatten::kernel_to_chunks(pdbClient, dbName, kernelset, + "temp_kernel", block_x, block_y); + + #ifdef PROFILING_CONV2D + img_conv_flatten::verify_chunks(pdbClient, dbName, "temp_kernel"); + #endif + + img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp_kernel", + "temp_kernel1", block_x); + + #ifdef PROFILING_CONV2D + img_conv_flatten::verify_blocks(pdbClient, dbName, "temp_kernel1"); + #endif + + img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp_kernel1", + "temp_kernel2", block_x, block_y, + block_padding, numOfFilters, kWidth * kHeight * kChannels + 1); + + #ifdef PROFILING_CONV2D + img_conv_flatten::verify_matrices(pdbClient, dbName, "temp_kernel2"); + #endif + + // // kernel bias join + // std::cout << "Running kernel bias join..." << std::endl; + + + // kernel_conv_flatten::kernel_bias_join(pdbClient, dbName, "temp_kernel2", + // biasset, "kernel_flat"); + + #ifdef PROFILING_CONV2D + ff::print(pdbClient, dbName, "kernel_flat"); + #endif + } + auto kernel_flat_end = std::chrono::high_resolution_clock::now(); + + + // multiply + std::cout << "Running conv2d op..." << std::endl; + + auto conv2d_begin = std::chrono::high_resolution_clock::now(); + + test_common::conv2d_op(pdbClient, dbName, "temp_image2", "kernel_flat", "result", materializeModel); + + auto conv2d_end = std::chrono::high_resolution_clock::now(); + + #ifdef PROFILING_CONV2D + ff::print(pdbClient, dbName, "result"); + #endif + + + +/* test_common::conv2d_result_to_chunks(pdbClient, 100, dbName, "result", "result_chunked"); + +#ifdef PROFILING_CONV2D + img_conv_flatten::verify_chunks(pdbClient, dbName, "result_chunked"); +#endif + + img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "result_chunked", + "result_chunked1", 11236); + +#ifdef PROFILING_CONV2D + img_conv_flatten::verify_blocks(pdbClient, dbName, "result_chunked1"); +#endif + { + const pdb::UseTemporaryAllocationBlock tempBlock{2047 * 1024 * 1024}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, "result_chunked1"); + + pdb::Handle chonk = pdb::makeObject(106, 106, 64); + chonk->setInput(readA); + + Handle myWriteSet = + makeObject>(dbName, "result_chunked2"); + myWriteSet->setInput(chonk); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "result-img", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + } + +#ifdef PROFILING_CONV2D + pdbClient.flushData(errMsg); + + img_conv_flatten::verify_data(pdbClient, dbName, "result_chunked2"); +#endif + + auto result_end = std::chrono::high_resolution_clock::now(); +*/ + + std::cout << "Total Time Duration : " + << std::chrono::duration_cast>(image_end - image_begin + kernel_flat_end - kernel_flat_begin + conv2d_end - conv2d_begin).count() + << " secs." << std::endl; + + std::cout << "Time Duration for image ops: " + << std::chrono::duration_cast>(image_end - image_begin).count() + << " secs." << std::endl; + + std::cout << "Time Duration for kernel flatterning ops: " + << std::chrono::duration_cast>(kernel_flat_end - kernel_flat_begin).count() + << " secs." << std::endl; + + std::cout << "Time Duration for conv2d: " + << std::chrono::duration_cast>(conv2d_end - conv2d_begin).count() + << " secs." << std::endl; + + std::cout << "Time Duration for loadRandomImages: " + << std::chrono::duration_cast>(loadRandomImages_end - loadRandomImages_time).count() + << " secs." << std::endl; + std::cout << "Time Duration for img_to_chunks: " + << std::chrono::duration_cast>(img_to_chunks_time - image_begin).count() + << " secs." << std::endl; + + std::cout << "Time Duration for verify_chunks: " + << std::chrono::duration_cast>(verify_chunks_time_end - img_to_chunks_time).count() + << " secs." << std::endl; + + std::cout << "Time Duration for chunks_to_blocks: " + << std::chrono::duration_cast>(chunks_to_blocks_end - verify_chunks_time_end).count() + << " secs." << std::endl; + + std::cout << "Time Duration for verify_blocks: " + << std::chrono::duration_cast>(verify_blocks_end - chunks_to_blocks_end).count() + << " secs." << std::endl; + + std::cout << "Time Duration for blocks_to_matrix: " + << std::chrono::duration_cast>(blocks_to_matrix_end - verify_blocks_end).count() + << " secs." << std::endl; + //std::cout << "Time Duration for result gathering: " + // << std::chrono::duration_cast>(result_end - result_begin).count() + // << " secs." << std::endl; + +} +vector splitString(string s) { + vector result; + stringstream s_stream(s); //create string stream from the string + while(s_stream.good()) { + string substr; + getline(s_stream, substr, ','); //get first string delimited by comma + result.push_back(substr); + } + + return result; +} + +int main(int argc, char *argv[]) { + + string masterIp = "localhost"; + pdb::PDBLoggerPtr clientLogger = make_shared("Conv2DclientLog"); + pdb::PDBClient pdbClient(8108, masterIp, clientLogger, false, true); + pdb::CatalogClient catalogClient(8108, masterIp, clientLogger); + + string errMsg; + + bool reloadData = true; + if (argc > 1) { + if (strcmp(argv[1], "N")==0) { + reloadData = false; + } + } + + bool materializeModel = false; + if (argc > 2) { + if (strcmp(argv[2], "Y")==0) { + materializeModel = true; + } + } + + string inputDimensions = "1,64,112,112"; + string kernelDimensions = "64,64,1,1"; + int n, c, w, h; + int nk, ck, wk, hk; + + vector imageDims = (argc > 3 )? splitString(argv[3]) : splitString(inputDimensions); + if (imageDims.size() != 4) { + std::cout << "Invalid format! Accepted format: N, C, W, H" << std::endl; + exit(1); + } + + vector kernelDims = (argc > 4 )? splitString(argv[4]) : splitString(kernelDimensions); + if (kernelDims.size() != 4) { + std::cout << "Invalid format! Accepted format: N, C, W, H" << std::endl; + exit(1); + } + // nk = std::stoi(kernelDims.at(0)); + // ck = std::stoi(kernelDims.at(1)); + // wk = std::stoi(kernelDims.at(2)); + // hk = std::stoi(kernelDims.at(3)); + + + // n = std::stoi(imageDims.at(0)); + // c = std::stoi(imageDims.at(1)); + // w = std::stoi(imageDims.at(2)); + // h = std::stoi(imageDims.at(3)) + + // Load Libraries + + if (reloadData) { + + if (!pdbClient.registerType("libraries/libFFMatrixData.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libFFMatrixData.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libFFMatrixMeta.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libFFMatrixMeta.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libFFMatrixBlock.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libFFMatrixBlock.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseMatrix3D.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseMatrix3D.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseImage.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseImage.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseImageChunk.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseImageChunk.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseImageToChunks.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseImageToChunks.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseImageBlock.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseImageBlock.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseImageChunksToBlock.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseImageChunksToBlock.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseImageBlockToMatrix.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseImageBlockToMatrix.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseKernel.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseKernel.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseKernelToChunks.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseKernelToChunks.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseKernelBiasJoin.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseKernelBiasJoin.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libFFTransposeMult.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libFFTransposeMult.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libFFAggMatrix.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libFFAggMatrix.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseConvResultToChunks.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseConvResultToChunks.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseConvChunksToImage.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseConvChunksToImage.so" + << ": " << errMsg << endl; + exit(-1); + } + } + + string dbName = "conv2d"; + string img_set = "img"; + string kernel_set = "kernel"; + test_conv2d_multiply(pdbClient, dbName, img_set, kernel_set, "bias", reloadData, materializeModel, imageDims, kernelDims); + + return 0; +} + diff --git a/src/tests/source/Conv2dMemFuseTest1.cc b/src/tests/source/Conv2dMemFuseTest1.cc index 5ae3c845..34664d45 100644 --- a/src/tests/source/Conv2dMemFuseTest1.cc +++ b/src/tests/source/Conv2dMemFuseTest1.cc @@ -1,916 +1,917 @@ -#include -#include -#include - -#include "FFMatrixBlock.h" -#include "FFMatrixData.h" -#include "FFMatrixMeta.h" - -#include "Image.h" -#include "ImageBlock.h" -#include "ImageBlockToMatrix.h" -#include "ImageChunk.h" -#include "ImageChunksToBlock.h" -#include "ImageToChunks.h" -#include "Matrix3D.h" - -#include "Kernel.h" -#include "KernelBiasJoin.h" -#include "KernelToChunks.h" -#include "ConvResultToChunks.h" -#include "ConvChunksToImage.h" - -#include "FFMatrixUtil.h" -#include "ImageUtils.h" - -#include "FFTransposeMult.h" -#include "FFAggMatrix.h" - -#include "PDBClient.h" -#include "PDBVector.h" -#include "ScanUserSet.h" -#include "WriteUserSet.h" -using namespace std; - -void load_rnd_img(int x, int y, int z, int size, pdb::PDBClient &pdbClient, - pdb::String dbName, pdb::String setName) { - std::string errMsg; - pdb::makeObjectAllocatorBlock(size * 1024 * 1024, true); - - pdb::Handle>> storeImage = - pdb::makeObject>>(); - - pdb::Handle myData = - pdb::makeObject(0, x, y, z); - - std::random_device rd; - - std::mt19937 e2(rd()); - - std::uniform_real_distribution<> distp(0.0001, 0.5); - std::uniform_real_distribution<> distn(-0.5, -0.0001); - - auto gen = std::bind(std::uniform_int_distribution<>(0, 1), - std::default_random_engine()); - - try { - for (int c = 0; c < z; c++) { - - pdb::Handle chan = - pdb::makeObject(0, 0, x, y, x, y); - - for (int i = 0; i < x; i++) { - for (int j = 0; j < y; j++) { - double data = (bool)gen() ? distn(e2) : distp(e2); - (*(chan->getRawDataHandle()))[i * y + j] = data; - } - } - - myData->addChannel(chan); - } - - storeImage->push_back(myData); - - if (!pdbClient.sendData( - pair(setName, dbName), storeImage, errMsg)) { - cout << "Failed to send data to dispatcher server" << endl; - exit(1); - } - } catch (pdb::NotEnoughSpace &e) { - cout << "Failed to send data to dispatcher server" << endl; - exit(1); - } - - // to write back all buffered records - pdbClient.flushData(errMsg); -} - -namespace test_common { -void create_database(pdb::PDBClient &pdbClient, std::string dbName, - std::vector sets, bool whetherToCreateDatabase=true) { - std::string errMsg; - std::cout << "Setting up database..." << std::endl; - - if (whetherToCreateDatabase) { - pdbClient.removeDatabase(dbName, errMsg); - if (!pdbClient.createDatabase(dbName, errMsg)) { - cout << "Not able to create database " << dbName << ": " << errMsg << endl; - } else { - cout << "Created database " << dbName << endl; - } - } - for (auto &s : sets) { - pdbClient.removeSet(dbName, s, errMsg); - if (!pdbClient.createSet( - dbName, s, errMsg, (size_t)128 * (size_t)1024 * (size_t)1024, s)) { - cout << "Not able to create set " + s + ": " + errMsg; - } else { - cout << "Created set " << s << ".\n"; - } - } -} - -void conv2d_op(pdb::PDBClient &pdbClient, std::string dbName, std::string image, - std::string kernel, std::string res) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, image); - pdb::Handle readB = - makeObject>(dbName, kernel); - - pdb::Handle join = pdb::makeObject(); - join->setInput(0, readA); - join->setInput(1, readB); - - // make the aggregation - pdb::Handle myAggregation = pdb::makeObject(); - myAggregation->setInput(join); - - Handle myWriteSet = - makeObject>(dbName, res); - myWriteSet->setInput(myAggregation); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "conv2d", true, myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - //pdbClient.flushData(errMsg); -} - -void conv2d_result_to_chunks(pdb::PDBClient &pdbClient, int images, std::string dbName, std::string result, std::string to) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, result); - - pdb::Handle chonk = pdb::makeObject(images); - chonk->setInput(readA); - - Handle myWriteSet = - makeObject>(dbName, to); - myWriteSet->setInput(chonk); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "result-chonk", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - //pdbClient.flushData(errMsg); -} -} // namespace test_common - -namespace img_conv_flatten { -void verify_data(pdb::PDBClient &pdbClient, std::string dbName, - std::string setName) { - std::cout << "Verifying data..." << std::endl; - - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - auto it = pdbClient.getSetIterator(dbName, setName); - - for (auto r : it) { - for (int c = 0; c < r->get_num_channels(); c++) { - pdb::Handle chan = r->getMatrixAtIndex(c); - std::cout << "Channel " << c << std::endl; - double *data = chan->getValue().rawData->c_ptr(); - for (int i = 0; i < r->x * r->y; i++) { - std::cout << data[i] << ","; - } - std::cout << std::endl; - } - std::cout << std::endl; - } -} - -void verify_chunks(pdb::PDBClient &pdbClient, std::string dbName, - std::string setName) { - std::cout << "Verifying chunks..." << std::endl; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - auto it = pdbClient.getSetIterator(dbName, setName); - - for (auto r : it) { - r->dump(); - } -} - -void verify_blocks(pdb::PDBClient &pdbClient, std::string dbName, - std::string setName) { - std::cout << "Verifying blocks..." << std::endl; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - auto it = pdbClient.getSetIterator(dbName, setName); - - for (auto r : it) { - r->dump(); - } -} - -void verify_matrices(pdb::PDBClient &pdbClient, std::string dbName, - std::string setName) { - std::cout << "Verifying matrices..." << std::endl; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - auto it = pdbClient.getSetIterator(dbName, setName); - - for (auto r : it) { - std::cout << "Block " << r->getBlockRowIndex() << "," - << r->getBlockColIndex() << std::endl; - double *data = r->getValue().rawData->c_ptr(); - for (int i = 0; i < r->getColNums() * r->getRowNums(); i++) { - std::cout << data[i] << ","; - } - std::cout << std::endl; - } -} - -void img_to_chunks(pdb::PDBClient &pdbClient, std::string dbName, - std::string from, std::string to, int block_x, int block_y, - int strides, int kernel, int padding) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, from); - - pdb::Handle chunk = - pdb::makeObject(block_y, kernel, strides, padding); - chunk->setInput(readA); - - Handle myWriteSet = - makeObject>(dbName, to); - myWriteSet->setInput(chunk); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "chunk", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - //pdbClient.flushData(errMsg); -} - -void chunks_to_blocks(pdb::PDBClient &pdbClient, std::string dbName, - std::string from, std::string to, int block_x) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, from); - - pdb::Handle block = - pdb::makeObject(block_x); - block->setInput(readA); - - Handle myWriteSet = - makeObject>(dbName, to); - myWriteSet->setInput(block); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "chunk1", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - //pdbClient.flushData(errMsg); -} - -void blocks_to_matrix(pdb::PDBClient &pdbClient, std::string dbName, - std::string from, std::string to, int block_x, - int block_y, bool block_padding, int X, int Y) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, from); - - pdb::Handle matrix = - pdb::makeObject(block_x, block_y, - block_padding, X, Y); - matrix->setInput(readA); - - Handle myWriteSet = - makeObject>(dbName, to); - myWriteSet->setInput(matrix); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "chunk2", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - //pdbClient.flushData(errMsg); -} -} // namespace img_conv_flatten - -namespace kernel_conv_flatten { -void kernel_to_chunks(pdb::PDBClient &pdbClient, std::string dbName, - std::string from, std::string to, int block_x, - int block_y) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, from); - - pdb::Handle chunk = - pdb::makeObject(block_y); - chunk->setInput(readA); - - Handle myWriteSet = - makeObject>(dbName, to); - myWriteSet->setInput(chunk); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "chunk", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - //pdbClient.flushData(errMsg); -} - -void kernel_bias_join(pdb::PDBClient &pdbClient, std::string dbName, - std::string a, std::string b, std::string to) { - std::string errMsg; - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - // make the computation - pdb::Handle readA = - makeObject>(dbName, a); - pdb::Handle readB = - makeObject>(dbName, b); - - pdb::Handle join = - pdb::makeObject(); - join->setInput(0, readA); - join->setInput(1, readB); - - Handle myWriteSet = - makeObject>(dbName, to); - myWriteSet->setInput(join); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "kernel_bias_join", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - //pdbClient.flushData(errMsg); -} -} // namespace kernel_conv_flatten - -void test_single_img_conv_flatten(pdb::PDBClient &pdbClient, std::string dbName, - std::string img_set) { - std::vector sets{img_set, "temp", "temp1", "temp2"}; - - std::string errMsg; - - test_common::create_database(pdbClient, dbName, sets); - - std::cout << "Loading data..." << std::endl; - - int channels = 2; - load_rnd_img(6, 6, channels, 64, pdbClient, dbName, img_set); - img_conv_flatten::verify_data(pdbClient, dbName, img_set); - - int block_x = 20; - int block_y = 20; - int kernel = 3; - int strides = 1; - int padding = 2; - bool block_padding = true; - - img_conv_flatten::img_to_chunks(pdbClient, dbName, img_set, "temp", block_x, - block_y, strides, kernel, padding); - img_conv_flatten::verify_chunks(pdbClient, dbName, "temp"); - - img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp", "temp1", - block_x); - img_conv_flatten::verify_blocks(pdbClient, dbName, "temp1"); - - img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp1", "temp2", - block_x, block_y, block_padding, 64, - 18 + 1); - img_conv_flatten::verify_matrices(pdbClient, dbName, "temp2"); -} - -void test_multiple_img_conv_flatten(pdb::PDBClient &pdbClient, - std::string dbName, std::string img_set) { - std::vector sets{img_set, "temp", "temp1", "temp2"}; - - std::string errMsg; - - test_common::create_database(pdbClient, dbName, sets); - - std::cout << "Loading data..." << std::endl; - - int channels = 2; - conv2d_memory_fusion::load_imgs_from_file( - pdbClient, "/usr/data/conv2d/images_1.np", dbName, img_set, 1, 2, 6, 6); - img_conv_flatten::verify_data(pdbClient, dbName, img_set); - - int block_x = 20; - int block_y = 20; - int kernel = 3; - int strides = 1; - int padding = 2; - bool block_padding = true; - - img_conv_flatten::img_to_chunks(pdbClient, dbName, img_set, "temp", block_x, - block_y, strides, kernel, padding); - img_conv_flatten::verify_chunks(pdbClient, dbName, "temp"); - - img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp", "temp1", - block_x); - img_conv_flatten::verify_blocks(pdbClient, dbName, "temp1"); - - img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp1", "temp2", - block_x, block_y, block_padding, 64, - 18 + 1); - img_conv_flatten::verify_matrices(pdbClient, dbName, "temp2"); -} - -void test_kernel_conv_flatten(pdb::PDBClient &pdbClient, std::string dbName, - std::string setName) { - std::vector sets{setName, "temp", "temp1", - "temp2", "bias", "kernel_flat"}; - std::string errMsg; - - test_common::create_database(pdbClient, dbName, sets); - - std::cout << "Loading data..." << std::endl; - conv2d_memory_fusion::load_imgs_from_file( - pdbClient, "/usr/data/conv2d/kernel_3.np", dbName, setName, 3, 2, 3, 3); - img_conv_flatten::verify_data(pdbClient, dbName, setName); - - int block_x = 10; - int block_y = 10; - bool block_padding = true; - ff::load_matrix_data(pdbClient, "/usr/data/conv2d/bias_3.np", dbName, "bias", - block_x, 1, !block_padding, !block_padding, errMsg); - ff::print(pdbClient, dbName, "bias"); - - kernel_conv_flatten::kernel_to_chunks(pdbClient, dbName, setName, "temp", - block_x, block_y); - img_conv_flatten::verify_chunks(pdbClient, dbName, "temp"); - - img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp", "temp1", - block_x); - img_conv_flatten::verify_blocks(pdbClient, dbName, "temp1"); - - img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp1", "temp2", - block_x, block_y, block_padding, 3, - 18 + 1); - img_conv_flatten::verify_matrices(pdbClient, dbName, "temp2"); - - kernel_conv_flatten::kernel_bias_join(pdbClient, dbName, "temp2", "bias", - "kernel_flat"); - ff::print(pdbClient, dbName, "kernel_flat"); -} - -void test_conv2d_multiply(pdb::PDBClient &pdbClient, std::string dbName, - std::string imageset, std::string kernelset, - std::string biasset, bool reloadData) { - std::vector allSets{kernelset, biasset, "temp_kernel", "temp_kernel1", "temp_kernel2", "kernel_flat", "temp_image", "temp_image1", "temp_image2", - "result", "result_chunked", "result_chunked1", "result_chunked2"}; - - std::vector intermediateSets{"temp_image", "temp_image1", "temp_image2", - "result", "result_chunked", "result_chunked1", "result_chunked2"}; - - std::string errMsg; - - int block_x = 100; - int block_y = 100; - int kernel = 7;//kernel shape should be $kernelx$kernel (e.g., 7x7) - int strides = 1; - int padding = 0; - bool block_padding = true; - int channels = 3; - - if (reloadData) { - - test_common::create_database(pdbClient, dbName, allSets, reloadData); - - pdbClient.removeSet(dbName, imageset, errMsg); - if (!pdbClient.createSet( - dbName, imageset, errMsg, (size_t)2 * (size_t)1024 * (size_t)1024, imageset)) { - cout << "Not able to create set " + imageset + ": " + errMsg; - } else { - cout << "Created set " << imageset << ".\n"; - } - - - - std::cout << "Loading image data..." << std::endl; - conv2d_memory_fusion::load_imgs_from_file( - // pdbClient, "/home/ubuntu/conv2d/images_1.np", dbName, imageset); - // pdbClient, "/home/ubuntu/images_10_3_112_112.np", dbName, imageset); - pdbClient, "/home/ubuntu/images_100_3_112_112.np", dbName, imageset, 100, 3, 112, 112); - // pdbClient, "/home/ubuntu/images_100.np", dbName, imageset); - // pdbClient, "/home/ubuntu/images_100_3_224_224.np", dbName, imageset); - -#ifdef PROFILING_CONV2D - img_conv_flatten::verify_data(pdbClient, dbName, imageset); -#endif - - std::cout << "Loading kernel data..." << std::endl; - conv2d_memory_fusion::load_imgs_from_file( - // pdbClient, "/home/ubuntu/conv2d/kernel_3.np", dbName, kernelset); - pdbClient, "/home/ubuntu/conv2d/kernel_64_3_7_7.np", dbName, kernelset, 64, 3, 7, 7); -#ifdef PROFILING_CONV2D - img_conv_flatten::verify_data(pdbClient, dbName, kernelset); -#endif - std::cout << "Loading bias data..." << std::endl; - //ff::load_matrix_data(pdbClient, "/home/ubuntu/conv2d/bias_3.np", dbName, biasset, - ff::load_matrix_data(pdbClient, "/home/ubuntu/conv2d/bias_64.np", dbName, biasset, - block_x, 1, !block_padding, !block_padding, errMsg); -#ifdef PROFILING_CONV2D - ff::print(pdbClient, dbName, biasset); -#endif - } - else { - - test_common::create_database(pdbClient, dbName, intermediateSets, reloadData); - } - - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 1024}; - - // Image ops - std::cout << "Running image ops..." << std::endl; - - auto image_begin = std::chrono::high_resolution_clock::now(); - - - - pdb::Handle readA = - makeObject>(dbName, imageset); - - pdb::Handle chunk = - pdb::makeObject(block_y, kernel, strides, padding); - - chunk->setInput(readA); - - - //img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp_image", - //"temp_image1", block_x); - - pdb::Handle block = - pdb::makeObject(block_x); - block->setInput(chunk); - - - - //X: num_rows per flattened image (112-7+1)(112-7+1) - //Y: num_cols per flattened image (7*7*3+1) why +1? - //img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp_image1", - //"temp_image2", block_x, block_y, - //block_padding, 1123600, 147 + 1); - - pdb::Handle matrix = - pdb::makeObject(block_x, block_y, - block_padding, 1123600, 147+1); - matrix->setInput(block); - - - Handle myWriteSet2 = - makeObject>(dbName, "temp_image2"); - - myWriteSet2->setInput(matrix); - - - // run the computation - if (!pdbClient.executeComputations(errMsg, "image_ops", myWriteSet2)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - -//#ifdef PROFILING_CONV2D - pdbClient.flushData(errMsg); - ff::print(pdbClient, dbName, "temp_image2"); -//#endif - - - auto image_end = std::chrono::high_resolution_clock::now(); - auto kernel_flat_begin = std::chrono::high_resolution_clock::now(); - if (reloadData) { - //if (true) { - // kernel ops - std::cout << "Running kernel ops..." << std::endl; - - //kernel_conv_flatten::kernel_to_chunks(pdbClient, dbName, kernelset, - //"temp_kernel", block_x, block_y); - - pdb::Handle readKernel = - makeObject>(dbName, kernelset); - - - pdb::Handle kernel_chunk = - pdb::makeObject(block_y); - kernel_chunk->setInput(readKernel); - - - //img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp_kernel", - //"temp_kernel1", block_x); - - pdb::Handle kernel_block = - pdb::makeObject(block_x); - kernel_block->setInput(kernel_chunk); - - - //img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp_kernel1", - //"temp_kernel2", block_x, block_y, - //block_padding, 64, 147 + 1); - - pdb::Handle kernel_matrix = - pdb::makeObject(block_x, block_y, - block_padding, 64, 147+1); - kernel_matrix->setInput(kernel_block); - - - // kernel bias join - std::cout << "Running kernel bias join..." << std::endl; - - - //kernel_conv_flatten::kernel_bias_join(pdbClient, dbName, "temp_kernel2", - // biasset, "kernel_flat"); - pdb::Handle readB = - makeObject>(dbName, biasset); - - pdb::Handle join1 = - pdb::makeObject(); - join1->setInput(0, kernel_matrix); - join1->setInput(1, readB); - - Handle myWriteSet = - makeObject>(dbName, "kernel_flat"); - myWriteSet->setInput(join1); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "kernel_bias_join", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - - - //#ifdef PROFILING_CONV2D - ff::print(pdbClient, dbName, "kernel_flat"); - //#endif - - - } - auto kernel_flat_end = std::chrono::high_resolution_clock::now(); - - - // multiply - std::cout << "Running conv2d op..." << std::endl; - - auto conv2d_begin = std::chrono::high_resolution_clock::now(); - - test_common::conv2d_op(pdbClient, dbName, "temp_image2", "kernel_flat", "result"); - - auto conv2d_end = std::chrono::high_resolution_clock::now(); - -//#ifdef PROFILING_CONV2D - ff::print(pdbClient, dbName, "result"); -//#endif - - // result gather -/* std::cout << "gathering results..." << std::endl; - - auto result_begin = std::chrono::high_resolution_clock::now(); - - test_common::conv2d_result_to_chunks(pdbClient, 100, dbName, "result", "result_chunked"); - -#ifdef PROFILING_CONV2D - img_conv_flatten::verify_chunks(pdbClient, dbName, "result_chunked"); -#endif - - img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "result_chunked", - "result_chunked1", 11236); - -#ifdef PROFILING_CONV2D - img_conv_flatten::verify_blocks(pdbClient, dbName, "result_chunked1"); -#endif - { - - // make the computation - pdb::Handle readA = - makeObject>(dbName, "result_chunked1"); - - pdb::Handle chonk = pdb::makeObject(106, 106, 64); - chonk->setInput(readA); - - Handle myWriteSet = - makeObject>(dbName, "result_chunked2"); - myWriteSet->setInput(chonk); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "result-img", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - } - -#ifdef PROFILING_CONV2D - pdbClient.flushData(errMsg); - - img_conv_flatten::verify_data(pdbClient, dbName, "result_chunked2"); -#endif - - auto result_end = std::chrono::high_resolution_clock::now(); - -*/ - - std::cout << "Time Duration for image ops: " - << std::chrono::duration_cast>(image_end - image_begin).count() - << " secs." << std::endl; - - std::cout << "Time Duration for kernel flatterning ops: " - << std::chrono::duration_cast>(kernel_flat_end - kernel_flat_begin).count() - << " secs." << std::endl; - - std::cout << "Time Duration for conv2d: " - << std::chrono::duration_cast>(conv2d_end - conv2d_begin).count() - << " secs." << std::endl; - - -} - -int main(int argc, char *argv[]) { - - - - string masterIp = "localhost"; - pdb::PDBLoggerPtr clientLogger = - make_shared("Conv2DclientLog"); - pdb::PDBClient pdbClient(8108, masterIp, clientLogger, false, true); - pdb::CatalogClient catalogClient(8108, masterIp, clientLogger); - - string errMsg; - string dbName = "conv2d"; - string img_set = "img"; - string kernel_set = "kernel"; - - bool reloadData = true; - if (argc > 1) { - if (strcmp(argv[1], "N")==0) { - reloadData = false; - } - } - // Load Libraries - - if (reloadData) { - - if (!pdbClient.registerType("libraries/libFFMatrixData.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libFFMatrixData.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libFFMatrixMeta.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libFFMatrixMeta.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libFFMatrixBlock.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libFFMatrixBlock.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseMatrix3D.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseMatrix3D.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseImage.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseImage.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseImageChunk.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseImageChunk.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseImageToChunks.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseImageToChunks.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseImageBlock.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseImageBlock.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseImageChunksToBlock.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseImageChunksToBlock.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseImageBlockToMatrix.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseImageBlockToMatrix.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseKernel.so", errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseKernel.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseKernelToChunks.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseKernelToChunks.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseKernelBiasJoin.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseKernelBiasJoin.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libFFTransposeMult.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libFFTransposeMult.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libFFAggMatrix.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libFFAggMatrix.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseConvResultToChunks.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseConvResultToChunks.so" - << ": " << errMsg << endl; - exit(-1); - } - - if (!pdbClient.registerType("libraries/libConv2DMemFuseConvChunksToImage.so", - errMsg)) { - cout << "Couldnt include " - << "libraries/libConv2DMemFuseConvChunksToImage.so" - << ": " << errMsg << endl; - exit(-1); - } - } - - // test_single_img_conv_flatten(pdbClient, dbName, img_set); - // test_multiple_img_conv_flatten(pdbClient, dbName, img_set); - // test_kernel_conv_flatten(pdbClient, dbName, kernel_set); - test_conv2d_multiply(pdbClient, dbName, img_set, kernel_set, "bias", reloadData); - - return 0; -} - - +#include +#include +#include +#include + +#include "FFMatrixBlock.h" +#include "FFMatrixData.h" +#include "FFMatrixMeta.h" + +#include "Image.h" +#include "ImageBlock.h" +#include "ImageBlockToMatrix.h" +#include "ImageChunk.h" +#include "ImageChunksToBlock.h" +#include "ImageToChunks.h" +#include "Matrix3D.h" + +#include "Kernel.h" +#include "KernelBiasJoin.h" +#include "KernelToChunks.h" +#include "ConvResultToChunks.h" +#include "ConvChunksToImage.h" + +#include "FFMatrixUtil.h" +#include "ImageUtils.h" + +#include "FFTransposeMult.h" +#include "FFAggMatrix.h" + +#include "PDBClient.h" +#include "PDBVector.h" +#include "ScanUserSet.h" +#include "WriteUserSet.h" +using namespace std; + +void load_rnd_img(int x, int y, int z, int size, pdb::PDBClient &pdbClient, + pdb::String dbName, pdb::String setName) { + std::string errMsg; + pdb::makeObjectAllocatorBlock(size * 1024 * 1024, true); + + pdb::Handle>> storeImage = + pdb::makeObject>>(); + + pdb::Handle myData = + pdb::makeObject(0, x, y, z); + + std::random_device rd; + + std::mt19937 e2(rd()); + + std::uniform_real_distribution<> distp(0.0001, 0.5); + std::uniform_real_distribution<> distn(-0.5, -0.0001); + + auto gen = std::bind(std::uniform_int_distribution<>(0, 1), + std::default_random_engine()); + + try { + for (int c = 0; c < z; c++) { + + pdb::Handle chan = + pdb::makeObject(0, 0, x, y, x, y); + + for (int i = 0; i < x; i++) { + for (int j = 0; j < y; j++) { + float data = (bool)gen() ? distn(e2) : distp(e2); + (*(chan->getRawDataHandle()))[i * y + j] = data; + } + } + + myData->addChannel(chan); + } + + storeImage->push_back(myData); + + if (!pdbClient.sendData( + pair(setName, dbName), storeImage, errMsg)) { + cout << "Failed to send data to dispatcher server" << endl; + exit(1); + } + } catch (pdb::NotEnoughSpace &e) { + cout << "Failed to send data to dispatcher server" << endl; + exit(1); + } + + // to write back all buffered records + pdbClient.flushData(errMsg); +} + +namespace test_common { +void create_database(pdb::PDBClient &pdbClient, std::string dbName, + std::vector sets, bool whetherToCreateDatabase=true) { + std::string errMsg; + std::cout << "Setting up database..." << std::endl; + + if (whetherToCreateDatabase) { + pdbClient.removeDatabase(dbName, errMsg); + if (!pdbClient.createDatabase(dbName, errMsg)) { + cout << "Not able to create database " << dbName << ": " << errMsg << endl; + } else { + cout << "Created database " << dbName << endl; + } + } + for (auto &s : sets) { + pdbClient.removeSet(dbName, s, errMsg); + if (!pdbClient.createSet( + dbName, s, errMsg, (size_t)128 * (size_t)1024 * (size_t)1024, s)) { + cout << "Not able to create set " + s + ": " + errMsg; + } else { + cout << "Created set " << s << ".\n"; + } + } +} + +void conv2d_op(pdb::PDBClient &pdbClient, std::string dbName, std::string image, + std::string kernel, std::string res) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, image); + pdb::Handle readB = + makeObject>(dbName, kernel); + + pdb::Handle join = pdb::makeObject(); + join->setInput(0, readA); + join->setInput(1, readB); + + // make the aggregation + pdb::Handle myAggregation = pdb::makeObject(); + myAggregation->setInput(join); + + Handle myWriteSet = + makeObject>(dbName, res); + myWriteSet->setInput(myAggregation); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "conv2d", true, myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + //pdbClient.flushData(errMsg); +} + +void conv2d_result_to_chunks(pdb::PDBClient &pdbClient, int images, std::string dbName, std::string result, std::string to) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, result); + + pdb::Handle chonk = pdb::makeObject(images); + chonk->setInput(readA); + + Handle myWriteSet = + makeObject>(dbName, to); + myWriteSet->setInput(chonk); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "result-chonk", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + //pdbClient.flushData(errMsg); +} +} // namespace test_common + +namespace img_conv_flatten { +void verify_data(pdb::PDBClient &pdbClient, std::string dbName, + std::string setName) { + std::cout << "Verifying data..." << std::endl; + + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + auto it = pdbClient.getSetIterator(dbName, setName); + + for (auto r : it) { + for (int c = 0; c < r->get_num_channels(); c++) { + pdb::Handle chan = r->getMatrixAtIndex(c); + std::cout << "Channel " << c << std::endl; + float *data = chan->getValue().rawData->c_ptr(); + for (int i = 0; i < r->x * r->y; i++) { + std::cout << data[i] << ","; + } + std::cout << std::endl; + } + std::cout << std::endl; + } +} + +void verify_chunks(pdb::PDBClient &pdbClient, std::string dbName, + std::string setName) { + std::cout << "Verifying chunks..." << std::endl; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + auto it = pdbClient.getSetIterator(dbName, setName); + + for (auto r : it) { + r->dump(); + } +} + +void verify_blocks(pdb::PDBClient &pdbClient, std::string dbName, + std::string setName) { + std::cout << "Verifying blocks..." << std::endl; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + auto it = pdbClient.getSetIterator(dbName, setName); + + for (auto r : it) { + r->dump(); + } +} + +void verify_matrices(pdb::PDBClient &pdbClient, std::string dbName, + std::string setName) { + std::cout << "Verifying matrices..." << std::endl; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + auto it = pdbClient.getSetIterator(dbName, setName); + + for (auto r : it) { + std::cout << "Block " << r->getBlockRowIndex() << "," + << r->getBlockColIndex() << std::endl; + float *data = r->getValue().rawData->c_ptr(); + for (int i = 0; i < r->getColNums() * r->getRowNums(); i++) { + std::cout << data[i] << ","; + } + std::cout << std::endl; + } +} + +void img_to_chunks(pdb::PDBClient &pdbClient, std::string dbName, + std::string from, std::string to, int block_x, int block_y, + int strides, int kernel, int padding) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, from); + + pdb::Handle chunk = + pdb::makeObject(block_y, kernel, strides, padding); + chunk->setInput(readA); + + Handle myWriteSet = + makeObject>(dbName, to); + myWriteSet->setInput(chunk); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "chunk", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + //pdbClient.flushData(errMsg); +} + +void chunks_to_blocks(pdb::PDBClient &pdbClient, std::string dbName, + std::string from, std::string to, int block_x) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, from); + + pdb::Handle block = + pdb::makeObject(block_x); + block->setInput(readA); + + Handle myWriteSet = + makeObject>(dbName, to); + myWriteSet->setInput(block); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "chunk1", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + //pdbClient.flushData(errMsg); +} + +void blocks_to_matrix(pdb::PDBClient &pdbClient, std::string dbName, + std::string from, std::string to, int block_x, + int block_y, bool block_padding, int X, int Y) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, from); + + pdb::Handle matrix = + pdb::makeObject(block_x, block_y, + block_padding, X, Y); + matrix->setInput(readA); + + Handle myWriteSet = + makeObject>(dbName, to); + myWriteSet->setInput(matrix); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "chunk2", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + //pdbClient.flushData(errMsg); +} +} // namespace img_conv_flatten + +namespace kernel_conv_flatten { +void kernel_to_chunks(pdb::PDBClient &pdbClient, std::string dbName, + std::string from, std::string to, int block_x, + int block_y) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, from); + + pdb::Handle chunk = + pdb::makeObject(block_y); + chunk->setInput(readA); + + Handle myWriteSet = + makeObject>(dbName, to); + myWriteSet->setInput(chunk); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "chunk", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + //pdbClient.flushData(errMsg); +} + +void kernel_bias_join(pdb::PDBClient &pdbClient, std::string dbName, + std::string a, std::string b, std::string to) { + std::string errMsg; + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + // make the computation + pdb::Handle readA = + makeObject>(dbName, a); + pdb::Handle readB = + makeObject>(dbName, b); + + pdb::Handle join = + pdb::makeObject(); + join->setInput(0, readA); + join->setInput(1, readB); + + Handle myWriteSet = + makeObject>(dbName, to); + myWriteSet->setInput(join); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "kernel_bias_join", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + //pdbClient.flushData(errMsg); +} +} // namespace kernel_conv_flatten + +void test_single_img_conv_flatten(pdb::PDBClient &pdbClient, std::string dbName, + std::string img_set) { + std::vector sets{img_set, "temp", "temp1", "temp2"}; + + std::string errMsg; + + test_common::create_database(pdbClient, dbName, sets); + + std::cout << "Loading data..." << std::endl; + + int channels = 2; + load_rnd_img(6, 6, channels, 64, pdbClient, dbName, img_set); + img_conv_flatten::verify_data(pdbClient, dbName, img_set); + + int block_x = 20; + int block_y = 20; + int kernel = 3; + int strides = 1; + int padding = 2; + bool block_padding = true; + + img_conv_flatten::img_to_chunks(pdbClient, dbName, img_set, "temp", block_x, + block_y, strides, kernel, padding); + img_conv_flatten::verify_chunks(pdbClient, dbName, "temp"); + + img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp", "temp1", + block_x); + img_conv_flatten::verify_blocks(pdbClient, dbName, "temp1"); + + img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp1", "temp2", + block_x, block_y, block_padding, 64, + 18 + 1); + img_conv_flatten::verify_matrices(pdbClient, dbName, "temp2"); +} + +void test_multiple_img_conv_flatten(pdb::PDBClient &pdbClient, + std::string dbName, std::string img_set) { + std::vector sets{img_set, "temp", "temp1", "temp2"}; + + std::string errMsg; + + test_common::create_database(pdbClient, dbName, sets); + + std::cout << "Loading data..." << std::endl; + + int channels = 2; + conv2d_memory_fusion::load_imgs_from_file( + pdbClient, "/usr/data/conv2d/images_1.np", dbName, img_set, 1, 2, 6, 6); + img_conv_flatten::verify_data(pdbClient, dbName, img_set); + + int block_x = 20; + int block_y = 20; + int kernel = 3; + int strides = 1; + int padding = 2; + bool block_padding = true; + + img_conv_flatten::img_to_chunks(pdbClient, dbName, img_set, "temp", block_x, + block_y, strides, kernel, padding); + img_conv_flatten::verify_chunks(pdbClient, dbName, "temp"); + + img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp", "temp1", + block_x); + img_conv_flatten::verify_blocks(pdbClient, dbName, "temp1"); + + img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp1", "temp2", + block_x, block_y, block_padding, 64, + 18 + 1); + img_conv_flatten::verify_matrices(pdbClient, dbName, "temp2"); +} + +void test_kernel_conv_flatten(pdb::PDBClient &pdbClient, std::string dbName, + std::string setName) { + std::vector sets{setName, "temp", "temp1", + "temp2", "bias", "kernel_flat"}; + std::string errMsg; + + test_common::create_database(pdbClient, dbName, sets); + + std::cout << "Loading data..." << std::endl; + conv2d_memory_fusion::load_imgs_from_file( + pdbClient, "/usr/data/conv2d/kernel_3.np", dbName, setName, 3, 2, 3, 3); + img_conv_flatten::verify_data(pdbClient, dbName, setName); + + int block_x = 10; + int block_y = 10; + bool block_padding = true; + ff::load_matrix_data(pdbClient, "/usr/data/conv2d/bias_3.np", dbName, "bias", + block_x, 1, !block_padding, !block_padding, errMsg); + ff::print(pdbClient, dbName, "bias"); + + kernel_conv_flatten::kernel_to_chunks(pdbClient, dbName, setName, "temp", + block_x, block_y); + img_conv_flatten::verify_chunks(pdbClient, dbName, "temp"); + + img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp", "temp1", + block_x); + img_conv_flatten::verify_blocks(pdbClient, dbName, "temp1"); + + img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp1", "temp2", + block_x, block_y, block_padding, 3, + 18 + 1); + img_conv_flatten::verify_matrices(pdbClient, dbName, "temp2"); + + kernel_conv_flatten::kernel_bias_join(pdbClient, dbName, "temp2", "bias", + "kernel_flat"); + ff::print(pdbClient, dbName, "kernel_flat"); +} + +void test_conv2d_multiply(pdb::PDBClient &pdbClient, std::string dbName, + std::string imageset, std::string kernelset, + std::string biasset, bool reloadData) { + std::vector allSets{kernelset, biasset, "temp_kernel", "temp_kernel1", "temp_kernel2", "kernel_flat", "temp_image", "temp_image1", "temp_image2", + "result", "result_chunked", "result_chunked1", "result_chunked2"}; + + std::vector intermediateSets{"temp_image", "temp_image1", "temp_image2", + "result", "result_chunked", "result_chunked1", "result_chunked2"}; + + std::string errMsg; + + int block_x = 100; + int block_y = 100; + int kernel = 7;//kernel shape should be $kernelx$kernel (e.g., 7x7) + int strides = 1; + int padding = 0; + bool block_padding = true; + int channels = 3; + + if (reloadData) { + + test_common::create_database(pdbClient, dbName, allSets, reloadData); + + pdbClient.removeSet(dbName, imageset, errMsg); + if (!pdbClient.createSet( + dbName, imageset, errMsg, (size_t)2 * (size_t)1024 * (size_t)1024, imageset)) { + cout << "Not able to create set " + imageset + ": " + errMsg; + } else { + cout << "Created set " << imageset << ".\n"; + } + + + + std::cout << "Loading image data..." << std::endl; + conv2d_memory_fusion::load_imgs_from_file( + // pdbClient, "/home/ubuntu/conv2d/images_1.np", dbName, imageset); + // pdbClient, "/home/ubuntu/images_10_3_112_112.np", dbName, imageset); + pdbClient, "/home/ubuntu/images_100_3_112_112.np", dbName, imageset, 100, 3, 112, 112); + // pdbClient, "/home/ubuntu/images_100.np", dbName, imageset); + // pdbClient, "/home/ubuntu/images_100_3_224_224.np", dbName, imageset); + +#ifdef PROFILING_CONV2D + img_conv_flatten::verify_data(pdbClient, dbName, imageset); +#endif + + std::cout << "Loading kernel data..." << std::endl; + conv2d_memory_fusion::load_imgs_from_file( + // pdbClient, "/home/ubuntu/conv2d/kernel_3.np", dbName, kernelset); + pdbClient, "/home/ubuntu/conv2d/kernel_64_3_7_7.np", dbName, kernelset, 64, 3, 7, 7); +#ifdef PROFILING_CONV2D + img_conv_flatten::verify_data(pdbClient, dbName, kernelset); +#endif + std::cout << "Loading bias data..." << std::endl; + //ff::load_matrix_data(pdbClient, "/home/ubuntu/conv2d/bias_3.np", dbName, biasset, + ff::load_matrix_data(pdbClient, "/home/ubuntu/conv2d/bias_64.np", dbName, biasset, + block_x, 1, !block_padding, !block_padding, errMsg); +#ifdef PROFILING_CONV2D + ff::print(pdbClient, dbName, biasset); +#endif + } + else { + + test_common::create_database(pdbClient, dbName, intermediateSets, reloadData); + } + + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 1024}; + + // Image ops + std::cout << "Running image ops..." << std::endl; + + auto image_begin = std::chrono::high_resolution_clock::now(); + + + + pdb::Handle readA = + makeObject>(dbName, imageset); + + pdb::Handle chunk = + pdb::makeObject(block_y, kernel, strides, padding); + + chunk->setInput(readA); + + + //img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp_image", + //"temp_image1", block_x); + + pdb::Handle block = + pdb::makeObject(block_x); + block->setInput(chunk); + + + + //X: num_rows per flattened image (112-7+1)(112-7+1) + //Y: num_cols per flattened image (7*7*3+1) why +1? + //img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp_image1", + //"temp_image2", block_x, block_y, + //block_padding, 1123600, 147 + 1); + + pdb::Handle matrix = + pdb::makeObject(block_x, block_y, + block_padding, 1123600, 147+1); + matrix->setInput(block); + + + Handle myWriteSet2 = + makeObject>(dbName, "temp_image2"); + + myWriteSet2->setInput(matrix); + + + // run the computation + if (!pdbClient.executeComputations(errMsg, "image_ops", myWriteSet2)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + +//#ifdef PROFILING_CONV2D + pdbClient.flushData(errMsg); + ff::print(pdbClient, dbName, "temp_image2"); +//#endif + + + auto image_end = std::chrono::high_resolution_clock::now(); + auto kernel_flat_begin = std::chrono::high_resolution_clock::now(); + if (reloadData) { + //if (true) { + // kernel ops + std::cout << "Running kernel ops..." << std::endl; + + //kernel_conv_flatten::kernel_to_chunks(pdbClient, dbName, kernelset, + //"temp_kernel", block_x, block_y); + + pdb::Handle readKernel = + makeObject>(dbName, kernelset); + + + pdb::Handle kernel_chunk = + pdb::makeObject(block_y); + kernel_chunk->setInput(readKernel); + + + //img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "temp_kernel", + //"temp_kernel1", block_x); + + pdb::Handle kernel_block = + pdb::makeObject(block_x); + kernel_block->setInput(kernel_chunk); + + + //img_conv_flatten::blocks_to_matrix(pdbClient, dbName, "temp_kernel1", + //"temp_kernel2", block_x, block_y, + //block_padding, 64, 147 + 1); + + pdb::Handle kernel_matrix = + pdb::makeObject(block_x, block_y, + block_padding, 64, 147+1); + kernel_matrix->setInput(kernel_block); + + + // kernel bias join + std::cout << "Running kernel bias join..." << std::endl; + + + //kernel_conv_flatten::kernel_bias_join(pdbClient, dbName, "temp_kernel2", + // biasset, "kernel_flat"); + pdb::Handle readB = + makeObject>(dbName, biasset); + + pdb::Handle join1 = + pdb::makeObject(); + join1->setInput(0, kernel_matrix); + join1->setInput(1, readB); + + Handle myWriteSet = + makeObject>(dbName, "kernel_flat"); + myWriteSet->setInput(join1); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "kernel_bias_join", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + + + //#ifdef PROFILING_CONV2D + ff::print(pdbClient, dbName, "kernel_flat"); + //#endif + + + } + auto kernel_flat_end = std::chrono::high_resolution_clock::now(); + + + // multiply + std::cout << "Running conv2d op..." << std::endl; + + auto conv2d_begin = std::chrono::high_resolution_clock::now(); + + test_common::conv2d_op(pdbClient, dbName, "temp_image2", "kernel_flat", "result"); + + auto conv2d_end = std::chrono::high_resolution_clock::now(); + +//#ifdef PROFILING_CONV2D + ff::print(pdbClient, dbName, "result"); +//#endif + + // result gather +/* std::cout << "gathering results..." << std::endl; + + auto result_begin = std::chrono::high_resolution_clock::now(); + + test_common::conv2d_result_to_chunks(pdbClient, 100, dbName, "result", "result_chunked"); + +#ifdef PROFILING_CONV2D + img_conv_flatten::verify_chunks(pdbClient, dbName, "result_chunked"); +#endif + + img_conv_flatten::chunks_to_blocks(pdbClient, dbName, "result_chunked", + "result_chunked1", 11236); + +#ifdef PROFILING_CONV2D + img_conv_flatten::verify_blocks(pdbClient, dbName, "result_chunked1"); +#endif + { + + // make the computation + pdb::Handle readA = + makeObject>(dbName, "result_chunked1"); + + pdb::Handle chonk = pdb::makeObject(106, 106, 64); + chonk->setInput(readA); + + Handle myWriteSet = + makeObject>(dbName, "result_chunked2"); + myWriteSet->setInput(chonk); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "result-img", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + } + +#ifdef PROFILING_CONV2D + pdbClient.flushData(errMsg); + + img_conv_flatten::verify_data(pdbClient, dbName, "result_chunked2"); +#endif + + auto result_end = std::chrono::high_resolution_clock::now(); + +*/ + + std::cout << "Time Duration for image ops: " + << std::chrono::duration_cast>(image_end - image_begin).count() + << " secs." << std::endl; + + std::cout << "Time Duration for kernel flatterning ops: " + << std::chrono::duration_cast>(kernel_flat_end - kernel_flat_begin).count() + << " secs." << std::endl; + + std::cout << "Time Duration for conv2d: " + << std::chrono::duration_cast>(conv2d_end - conv2d_begin).count() + << " secs." << std::endl; + + +} + +int main(int argc, char *argv[]) { + + + + string masterIp = "localhost"; + pdb::PDBLoggerPtr clientLogger = + make_shared("Conv2DclientLog"); + pdb::PDBClient pdbClient(8108, masterIp, clientLogger, false, true); + pdb::CatalogClient catalogClient(8108, masterIp, clientLogger); + + string errMsg; + string dbName = "conv2d"; + string img_set = "img"; + string kernel_set = "kernel"; + + bool reloadData = true; + if (argc > 1) { + if (strcmp(argv[1], "N")==0) { + reloadData = false; + } + } + // Load Libraries + + if (reloadData) { + + if (!pdbClient.registerType("libraries/libFFMatrixData.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libFFMatrixData.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libFFMatrixMeta.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libFFMatrixMeta.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libFFMatrixBlock.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libFFMatrixBlock.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseMatrix3D.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseMatrix3D.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseImage.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseImage.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseImageChunk.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseImageChunk.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseImageToChunks.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseImageToChunks.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseImageBlock.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseImageBlock.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseImageChunksToBlock.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseImageChunksToBlock.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseImageBlockToMatrix.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseImageBlockToMatrix.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseKernel.so", errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseKernel.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseKernelToChunks.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseKernelToChunks.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseKernelBiasJoin.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseKernelBiasJoin.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libFFTransposeMult.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libFFTransposeMult.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libFFAggMatrix.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libFFAggMatrix.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseConvResultToChunks.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseConvResultToChunks.so" + << ": " << errMsg << endl; + exit(-1); + } + + if (!pdbClient.registerType("libraries/libConv2DMemFuseConvChunksToImage.so", + errMsg)) { + cout << "Couldnt include " + << "libraries/libConv2DMemFuseConvChunksToImage.so" + << ": " << errMsg << endl; + exit(-1); + } + } + + // test_single_img_conv_flatten(pdbClient, dbName, img_set); + // test_multiple_img_conv_flatten(pdbClient, dbName, img_set); + // test_kernel_conv_flatten(pdbClient, dbName, kernel_set); + test_conv2d_multiply(pdbClient, dbName, img_set, kernel_set, "bias", reloadData); + + return 0; +} + + diff --git a/src/tests/source/Conv2dProjTest.cc b/src/tests/source/Conv2dProjTest.cc index f8948af5..e98bbc29 100644 --- a/src/tests/source/Conv2dProjTest.cc +++ b/src/tests/source/Conv2dProjTest.cc @@ -1,247 +1,289 @@ -#include -#include -#include - -#include "Conv2DSelect.h" -#include "TensorData.h" -#include "PDBClient.h" -#include "PDBVector.h" -#include "ScanUserSet.h" -#include "WriteUserSet.h" - -using namespace std; - -void load_rnd_img(int x, int y, int z, int size, pdb::PDBClient &pdbClient, - pdb::String dbName, pdb::String setName) { - std::string errMsg; - pdb::makeObjectAllocatorBlock(64 * 1024 * 1024, true); - - pdb::Handle> dimensions = pdb::makeObject>(); - dimensions->push_back(z); - dimensions->push_back(y); - dimensions->push_back(x); - - - pdb::Handle>> images = pdb::makeObject>>(); - - std::random_device rd; - - std::mt19937 e2(rd()); - - std::uniform_real_distribution<> distp(0.0001, 0.5); - std::uniform_real_distribution<> distn(-0.5, -0.0001); - - auto gen = std::bind(std::uniform_int_distribution<>(0, 1), - std::default_random_engine()); - - - for (int i = 0; i < size; i++) { - try { - pdb::Handle image = pdb::makeObject(3, dimensions); - for (int c = 0; c < z; c++) { - for (int h = 0; h < y; h++) { - for (int w = 0; w < x; w++) { - double data = (bool)gen() ? distn(e2) : distp(e2); - (*(image->rawData))[c * y * x + h * x + w] = data; - } - } - } - images->push_back(image); - } catch (pdb::NotEnoughSpace &e) { - if (!pdbClient.sendData( - pair(setName, dbName), images, errMsg)) { - cout << "Failed to send data to dispatcher server" << endl; - exit(1); - } - i--; - pdb::makeObjectAllocatorBlock(64 * 1024 * 1024, true); - dimensions = pdb::makeObject>(); - dimensions->push_back(z); - dimensions->push_back(y); - dimensions->push_back(x); - images = pdb::makeObject>>(); - } - } - - if (!pdbClient.sendData( - pair(setName, dbName), images, errMsg)) { - cout << "Failed to send data to dispatcher server" << endl; - exit(1); - } - // to write back all buffered records - pdbClient.flushData(errMsg); -} - -int main(int argc, char *argv[]) { - - bool addDataOrNot = true; - - bool createSetOrNot = true; - - int numImages = 100; - - string mode = "aten-conv2d"; - //string mode = "eigen-spatial"; - - if ( argc > 1 ) { - - if ( strcmp( argv[1], "Y" ) == 0 ) { - - addDataOrNot = true; - - } else { - - addDataOrNot = false; - - } - } - - if ( argc > 2 ) { - - if ( strcmp( argv[2], "Y") == 0 ) { - - createSetOrNot = true; - - } else { - - createSetOrNot = false; - - } - - } - - if ( argc > 3 ) { - - numImages = atoi(argv[3]); - - } - - if ( argc > 4 ) { - - mode = argv[4]; - - } - - - string masterIp = "localhost"; - pdb::PDBLoggerPtr clientLogger = - make_shared("Conv2DclientLog"); - pdb::PDBClient pdbClient(8108, masterIp, clientLogger, false, true); - pdb::CatalogClient catalogClient(8108, masterIp, clientLogger); - - string errMsg; - string dbName = "conv2d"; - string img_set = "img"; - - if (createSetOrNot == true) { - // Load Libraries - pdbClient.registerType("libraries/libConv2DSelect.so", errMsg); - pdbClient.registerType("libraries/libTensorData.so", errMsg); - - //create input dataset - pdbClient.removeDatabase(dbName, errMsg); - if (!pdbClient.createDatabase(dbName, errMsg)) { - cout << "Not able to create database " << dbName << ": " << errMsg << endl; - } else { - cout << "Created database " << dbName << endl; - } - - pdbClient.removeSet(dbName, img_set, errMsg); - if (!pdbClient.createSet( - dbName, img_set, errMsg, (size_t)128 * (size_t)1024 * (size_t)1024)) { - cout << "Not able to create set " + img_set + ": " + errMsg; - } else { - cout << "Created set " << img_set << ".\n"; - } - } - - //load data - if (addDataOrNot == true) { - load_rnd_img(112, 112, 3, numImages, pdbClient, dbName, img_set); - } - - //create output dataset - string feature_out_set = "feature_map"; - pdbClient.removeSet(dbName, feature_out_set, errMsg); - if (!pdbClient.createSet( - dbName, feature_out_set, errMsg, (size_t)128 * (size_t)1024 * (size_t)1024)) { - cout << "Not able to create set " + feature_out_set + ": " + errMsg; - } else { - cout << "Created set " << feature_out_set << ".\n"; - } - - auto begin1 = std::chrono::high_resolution_clock::now(); - pdb::makeObjectAllocatorBlock(64 * 1024 * 1024, true); - - //create scan computation - pdb::Handle imageScanner = - pdb::makeObject>(dbName, img_set); - - pdb::Handle> dimensions = pdb::makeObject>(4); - unsigned int x=7, y=7, z=3, n=64; - dimensions->push_back(n); - dimensions->push_back(z); - dimensions->push_back(y); - dimensions->push_back(x); - - pdb::Handle kernel = pdb::makeObject(4, dimensions); - - for (unsigned int i = 0; i < n*z*y*x; i++) { - - (*(kernel->rawData))[i] = 0.5; - - } - - - //create select computation - // stride (across all dimensions) - pdb::Handle select = pdb::makeObject(kernel, mode, 1); - select->setInput(0, imageScanner); - std::cout << "select's output type: "<< select->getOutputType() << std::endl; - //create write computation - pdb::Handle myWriteSet = - pdb::makeObject>(dbName, feature_out_set); - myWriteSet->setInput(0, select); - - bool res = false; - - auto begin = std::chrono::high_resolution_clock::now(); - - // run the computation - if (!pdbClient.executeComputations(errMsg, "conv2d-proj", myWriteSet)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - auto end = std::chrono::high_resolution_clock::now(); - - //test the output - bool printResult = true; - if (printResult == true) { - std::cout << "to print result..." << std::endl; - pdb::SetIterator result = - pdbClient.getSetIterator(dbName, feature_out_set); - std::cout << "Query results: "; - int count = 0; - for (auto a : result) { - count++; - //std::cout << count << ":" << (*(a->dimensions))[0] << "," << (*(a->dimensions))[1] << "," << (*(a->dimensions))[2] << ";"; - } - std::cout << "output count:" << count << "\n"; - } - - - - //remove the output set - pdbClient.removeSet(dbName, feature_out_set, errMsg); - - std::cout << "Time Duration: " - << std::chrono::duration_cast>(end - begin).count() - << " secs." << std::endl; - - std::cout << "Translation Overhead: " - << std::chrono::duration_cast>(begin - begin1).count() - << " secs." << std::endl; - - return 0; -} +#include +#include +#include + +#include "Conv2DSelect.h" +#include "TensorData.h" +#include "PDBClient.h" +#include "PDBVector.h" +#include "ScanUserSet.h" +#include "WriteUserSet.h" + +using namespace std; + +void load_rnd_img(int x, int y, int z, int size, pdb::PDBClient &pdbClient, + pdb::String dbName, pdb::String setName) { + std::string errMsg; + pdb::makeObjectAllocatorBlock(512 * 1024 * 1024, true); + + pdb::Handle> dimensions = pdb::makeObject>(); + dimensions->push_back(z); + dimensions->push_back(y); + dimensions->push_back(x); + + + pdb::Handle>> images = pdb::makeObject>>(); + + std::random_device rd; + + std::mt19937 e2(rd()); + + std::uniform_real_distribution<> distp(0.0001, 0.5); + std::uniform_real_distribution<> distn(-0.5, -0.0001); + + auto gen = std::bind(std::uniform_int_distribution<>(0, 1), + std::default_random_engine()); + + + for (int i = 0; i < size; i++) { + try { + pdb::Handle image = pdb::makeObject(3, dimensions); + for (int c = 0; c < z; c++) { + for (int h = 0; h < y; h++) { + for (int w = 0; w < x; w++) { + double data = (bool)gen() ? distn(e2) : distp(e2); + (*(image->rawData))[c * y * x + h * x + w] = data; + } + } + } + images->push_back(image); + } catch (pdb::NotEnoughSpace &e) { + if (!pdbClient.sendData( + pair(setName, dbName), images, errMsg)) { + cout << "Failed to send data to dispatcher server" << endl; + exit(1); + } + i--; + pdb::makeObjectAllocatorBlock(512 * 1024 * 1024, true); + dimensions = pdb::makeObject>(); + dimensions->push_back(z); + dimensions->push_back(y); + dimensions->push_back(x); + images = pdb::makeObject>>(); + } + } + + if (!pdbClient.sendData( + pair(setName, dbName), images, errMsg)) { + cout << "Failed to send data to dispatcher server" << endl; + exit(1); + } + // to write back all buffered records + pdbClient.flushData(errMsg); +} + +vector splitString(string s) { + vector result; + stringstream s_stream(s); //create string stream from the string + while(s_stream.good()) { + string substr; + getline(s_stream, substr, ','); //get first string delimited by comma + result.push_back(substr); + } + + return result; +} + +int main(int argc, char *argv[]) { + + bool addDataOrNot = true; + + bool createSetOrNot = true; + + int numImages = 100; + + string inputDimensions = "1,64,112,112"; + string kernelDimensions = "64,64,1,1"; + int n, c, w, h; + int nk, ck, wk, hk; + + string mode = "aten-conv2d"; + //string mode = "eigen-spatial"; + + if ( argc > 1 ) { + + if ( strcmp( argv[1], "Y" ) == 0 ) { + + addDataOrNot = true; + + } else { + + addDataOrNot = false; + + } + } + + if ( argc > 2 ) { + + if ( strcmp( argv[2], "Y") == 0 ) { + + createSetOrNot = true; + + } else { + + createSetOrNot = false; + + } + + } + + if ( argc > 3 ) { + + numImages = atoi(argv[3]); + + } + + if ( argc > 4 ) { + + mode = argv[4]; + + } + + vector imageDims = (argc > 5 )? splitString(argv[5]) : splitString(inputDimensions); + if (imageDims.size() != 4) { + std::cout << "Invalid format! Accepted format: N, C, W, H" << std::endl; + exit(1); + } + n = std::stoi(imageDims.at(0)); + c = std::stoi(imageDims.at(1)); + w = std::stoi(imageDims.at(2)); + h = std::stoi(imageDims.at(3)); + + vector kernelDims = (argc > 6 )? splitString(argv[6]) : splitString(kernelDimensions); + if (kernelDims.size() != 4) { + std::cout << "Invalid format! Accepted format: N, C, W, H" << std::endl; + exit(1); + } + nk = std::stoi(kernelDims.at(0)); + ck = std::stoi(kernelDims.at(1)); + wk = std::stoi(kernelDims.at(2)); + hk = std::stoi(kernelDims.at(3)); + + int stride = (argc > 7) ? atoi(argv[7]) : 1; + + std::cout << "Input dimensions: " << n << " " << c << " " << w << " " << h << std::endl; + std::cout << "Kernel dimensions: " << nk << " " << ck << " " << wk << " " << hk << std::endl; + std::cout << "Stride: " << stride << std::endl; + + string masterIp = "localhost"; + pdb::PDBLoggerPtr clientLogger = + make_shared("Conv2DclientLog"); + pdb::PDBClient pdbClient(8108, masterIp, clientLogger, false, true); + pdb::CatalogClient catalogClient(8108, masterIp, clientLogger); + + string errMsg; + string dbName = "conv2d"; + string img_set = "img"; + + if (createSetOrNot == true) { + // Load Libraries + pdbClient.registerType("libraries/libConv2DSelect.so", errMsg); + pdbClient.registerType("libraries/libTensorData.so", errMsg); + + //create input dataset + pdbClient.removeDatabase(dbName, errMsg); + if (!pdbClient.createDatabase(dbName, errMsg)) { + cout << "Not able to create database " << dbName << ": " << errMsg << endl; + } else { + cout << "Created database " << dbName << endl; + } + + pdbClient.removeSet(dbName, img_set, errMsg); + if (!pdbClient.createSet( + dbName, img_set, errMsg, (size_t)128 * (size_t)1024 * (size_t)1024)) { + cout << "Not able to create set " + img_set + ": " + errMsg; + } else { + cout << "Created set " << img_set << ".\n"; + } + } + + //load data + if (addDataOrNot == true) { + load_rnd_img(w, h, c, n, pdbClient, dbName, img_set); + } + + //create output dataset + string feature_out_set = "feature_map"; + pdbClient.removeSet(dbName, feature_out_set, errMsg); + if (!pdbClient.createSet( + dbName, feature_out_set, errMsg, (size_t)128 * (size_t)1024 * (size_t)1024)) { + cout << "Not able to create set " + feature_out_set + ": " + errMsg; + } else { + cout << "Created set " << feature_out_set << ".\n"; + } + + auto begin1 = std::chrono::high_resolution_clock::now(); + pdb::makeObjectAllocatorBlock(64 * 1024 * 1024, true); + + //create scan computation + pdb::Handle imageScanner = + pdb::makeObject>(dbName, img_set); + + pdb::Handle> dimensions = pdb::makeObject>(4); +// unsigned int x=1, y=1, z=64, n=64; + dimensions->push_back(nk); + dimensions->push_back(ck); + dimensions->push_back(hk); + dimensions->push_back(wk); + + pdb::Handle kernel = pdb::makeObject(4, dimensions); + + for (unsigned int i = 0; i < nk*ck*hk*wk; i++) { + + (*(kernel->rawData))[i] = 0.5; + + } + + + //create select computation + // stride (across all dimensions) + pdb::Handle select = pdb::makeObject(kernel, mode, stride); + select->setInput(0, imageScanner); + std::cout << "select's output type: "<< select->getOutputType() << std::endl; + //create write computation + pdb::Handle myWriteSet = + pdb::makeObject>(dbName, feature_out_set); + myWriteSet->setInput(0, select); + + bool res = false; + + auto begin = std::chrono::high_resolution_clock::now(); + + // run the computation + if (!pdbClient.executeComputations(errMsg, "conv2d-proj", myWriteSet)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + auto end = std::chrono::high_resolution_clock::now(); + + //test the output + bool printResult = true; + if (printResult == true) { + std::cout << "to print result..." << std::endl; + pdb::SetIterator result = + pdbClient.getSetIterator(dbName, feature_out_set); + std::cout << "Query results: "; + int count = 0; + for (auto a : result) { + count++; + //std::cout << count << ":" << (*(a->dimensions))[0] << "," << (*(a->dimensions))[1] << "," << (*(a->dimensions))[2] << ";"; + } + std::cout << "output count:" << count << "\n"; + } + + + + //remove the output set + pdbClient.removeSet(dbName, feature_out_set, errMsg); + + std::cout << "Time Duration: " + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; + + std::cout << "Translation Overhead: " + << std::chrono::duration_cast>(begin - begin1).count() + << " secs." << std::endl; + + return 0; +} diff --git a/src/tests/source/FCProjTest.cc b/src/tests/source/FCProjTest.cc index 11560ee3..a4e49fb3 100644 --- a/src/tests/source/FCProjTest.cc +++ b/src/tests/source/FCProjTest.cc @@ -1,159 +1,159 @@ -#include -#include -#include -#include -#include -#include - -#include "PDBClient.h" -#include "SimpleFF.h" -#include "FFMatrixBlock.h" -#include "FFMatrixUtil.h" -#include "FullyConnectedNetwork.h" -#include "WriteUserSet.h" - -using namespace std; - -int main(int argc, char *argv[]) { - - bool reloadData = true; - string errMsg; - string input_path, labels_path; - int block_x, block_y, batch_size; - - if (argc < 3) { - cout << "Usage: blockDimensionX blockDimensionY batchSize numFeatures numNeurons numLabels YY" - "path/to/weights/and/bias(leave empty if generate random)" - << endl; - exit(-1); - } - - block_x = atoi(argv[1]); - block_y = atoi(argv[2]); - - batch_size = atoi(argv[3]); - - int features = atoi(argv[4]); - int hid_size = atoi(argv[5]); - int num_labels = atoi(argv[6]); - - - if (argc >= 7) { - if (strcmp(argv[7], "N")==0) { - reloadData = false; - std::cout << "WARNING: we do not reload data!" << std::endl; - } - } - cout << "Using block dimensions " << block_x << ", " << block_y << endl; - - bool generate = true; - - string masterIp = "localhost"; - pdb::PDBLoggerPtr clientLogger = make_shared("FFclientLog"); - pdb::PDBClient pdbClient(8108, masterIp, clientLogger, false, true); - pdb::CatalogClient catalogClient(8108, masterIp, clientLogger); - - - - if (reloadData) { - - ff::createDatabase(pdbClient, "ff"); - ff::setup(pdbClient, "ff"); - pdbClient.registerType("libraries/libFullyConnectedNetwork.so", errMsg); - ff::createSet(pdbClient, "ff", "inputs", "inputs", 64); - - } - - ff::createSet(pdbClient, "ff", "output", "Output", 64); - - - if (!generate && reloadData) { - input_path = string(argv[4]) + "/input.out"; - labels_path = string(argv[4]) + "/labels.out"; - - // load the input data - ff::load_matrix_data(pdbClient, input_path, "ff", "inputs", - block_x, block_y, false, false, errMsg); - } else if (reloadData) { - - // X x features_size = None x 5000 - std::cout << "To load matrix for ff:inputs" << std::endl; - ff::loadMatrix(pdbClient, "ff", "inputs", batch_size, features, block_x, - block_y, false, false, errMsg); - - } - - const pdb::UseTemporaryAllocationBlock tempBlock1{(size_t)6 * (size_t)1024 * (size_t)1024 * (size_t)1024}; - - pdb::Handle inputScanner = - pdb::makeObject>("ff", "inputs"); - - pdb::Handle fullyConnectedNetwork = - pdb::makeObject(features, num_labels, hid_size, batch_size); - - fullyConnectedNetwork->setInput(0, inputScanner); - - pdb::Handle writer = - pdb::makeObject>("ff", "output"); - - writer->setInput(0, fullyConnectedNetwork); - - auto begin = std::chrono::high_resolution_clock::now(); - // run the computation - if (!pdbClient.executeComputations(errMsg, "fc-proj", writer)) { - cout << "Computation failed. Message was: " << errMsg << "\n"; - exit(1); - } - - auto end = std::chrono::high_resolution_clock::now(); - std::cout << "*****FFTest End-to-End Time Duration: ****" - << std::chrono::duration_cast>(end - begin).count() - << " secs." << std::endl; - - - - vector> labels_test; - - if (!generate) - ff::load_matrix_from_file(labels_path, labels_test); - - int correct = 0; - { - const pdb::UseTemporaryAllocationBlock tempBlock{(size_t)128 * (size_t)1024 * (size_t)1024}; - - auto it = pdbClient.getSetIterator("ff", "output"); - - for (auto r : it) { - double *data = r->getRawDataHandle()->c_ptr(); - int i = 0; - int j = r->getBlockRowIndex() * r->getRowNums(); - while (i < r->getRowNums() * r->getColNums()) { - if (!generate && j >= labels_test.size()) - break; - // double a = exp(data[i]); - // double b = exp(data[i + 1]); - // double sum = a + b; - - cout << data[i] << ", " << data[i + 1] << endl; - - if (!generate) { - int pos1 = data[i] > data[i + 1] ? 0 : 1; - int pos2 = labels_test[j][0] > labels_test[j][1] ? 0 : 1; - - if (pos1 == pos2) - correct++; - } - - i += r->getColNums(); - j++; - } - } - - if (!generate) - cout << "Accuracy: " << correct << "/" << labels_test.size() << std::endl; - } - - sleep(20); - - return 0; -} +#include +#include +#include +#include +#include +#include + +#include "PDBClient.h" +#include "SimpleFF.h" +#include "FFMatrixBlock.h" +#include "FFMatrixUtil.h" +#include "FullyConnectedNetwork.h" +#include "WriteUserSet.h" + +using namespace std; + +int main(int argc, char *argv[]) { + + bool reloadData = true; + string errMsg; + string input_path, labels_path; + int block_x, block_y, batch_size; + + if (argc < 3) { + cout << "Usage: blockDimensionX blockDimensionY batchSize numFeatures numNeurons numLabels YY" + "path/to/weights/and/bias(leave empty if generate random)" + << endl; + exit(-1); + } + + block_x = atoi(argv[1]); + block_y = atoi(argv[2]); + + batch_size = atoi(argv[3]); + + int features = atoi(argv[4]); + int hid_size = atoi(argv[5]); + int num_labels = atoi(argv[6]); + + + if (argc >= 7) { + if (strcmp(argv[7], "N")==0) { + reloadData = false; + std::cout << "WARNING: we do not reload data!" << std::endl; + } + } + cout << "Using block dimensions " << block_x << ", " << block_y << endl; + + bool generate = true; + + string masterIp = "localhost"; + pdb::PDBLoggerPtr clientLogger = make_shared("FFclientLog"); + pdb::PDBClient pdbClient(8108, masterIp, clientLogger, false, true); + pdb::CatalogClient catalogClient(8108, masterIp, clientLogger); + + + + if (reloadData) { + + ff::createDatabase(pdbClient, "ff"); + ff::setup(pdbClient, "ff"); + pdbClient.registerType("libraries/libFullyConnectedNetwork.so", errMsg); + ff::createSet(pdbClient, "ff", "inputs", "inputs", 64); + + } + + ff::createSet(pdbClient, "ff", "output", "Output", 64); + + + if (!generate && reloadData) { + input_path = string(argv[4]) + "/input.out"; + labels_path = string(argv[4]) + "/labels.out"; + + // load the input data + ff::load_matrix_data(pdbClient, input_path, "ff", "inputs", + block_x, block_y, false, false, errMsg); + } else if (reloadData) { + + // X x features_size = None x 5000 + std::cout << "To load matrix for ff:inputs" << std::endl; + ff::loadMatrix(pdbClient, "ff", "inputs", batch_size, features, block_x, + block_y, false, false, errMsg); + + } + + const pdb::UseTemporaryAllocationBlock tempBlock1{(size_t)6 * (size_t)1024 * (size_t)1024 * (size_t)1024}; + + pdb::Handle inputScanner = + pdb::makeObject>("ff", "inputs"); + + pdb::Handle fullyConnectedNetwork = + pdb::makeObject(features, num_labels, hid_size, batch_size); + + fullyConnectedNetwork->setInput(0, inputScanner); + + pdb::Handle writer = + pdb::makeObject>("ff", "output"); + + writer->setInput(0, fullyConnectedNetwork); + + auto begin = std::chrono::high_resolution_clock::now(); + // run the computation + if (!pdbClient.executeComputations(errMsg, "fc-proj", writer)) { + cout << "Computation failed. Message was: " << errMsg << "\n"; + exit(1); + } + + auto end = std::chrono::high_resolution_clock::now(); + std::cout << "*****FFTest End-to-End Time Duration: ****" + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; + + + + vector> labels_test; + + if (!generate) + ff::load_matrix_from_file(labels_path, labels_test); + + int correct = 0; + { + const pdb::UseTemporaryAllocationBlock tempBlock{(size_t)128 * (size_t)1024 * (size_t)1024}; + + auto it = pdbClient.getSetIterator("ff", "output"); + + for (auto r : it) { + float *data = r->getRawDataHandle()->c_ptr(); + int i = 0; + int j = r->getBlockRowIndex() * r->getRowNums(); + while (i < r->getRowNums() * r->getColNums()) { + if (!generate && j >= labels_test.size()) + break; + // double a = exp(data[i]); + // double b = exp(data[i + 1]); + // double sum = a + b; + + cout << data[i] << ", " << data[i + 1] << endl; + + if (!generate) { + int pos1 = data[i] > data[i + 1] ? 0 : 1; + int pos2 = labels_test[j][0] > labels_test[j][1] ? 0 : 1; + + if (pos1 == pos2) + correct++; + } + + i += r->getColNums(); + j++; + } + } + + if (!generate) + cout << "Accuracy: " << correct << "/" << labels_test.size() << std::endl; + } + + sleep(20); + + return 0; +} diff --git a/src/tests/source/FFTest.cc b/src/tests/source/FFTest.cc index 6aa71c34..8bf08924 100644 --- a/src/tests/source/FFTest.cc +++ b/src/tests/source/FFTest.cc @@ -1,184 +1,184 @@ -#include -#include -#include -#include -#include -#include - -#include "PDBClient.h" - -#include "FFMatrixBlock.h" -#include "FFMatrixUtil.h" -#include "SimpleFF.h" - -using namespace std; - -int main(int argc, char *argv[]) { - - bool reloadData = true; - string errMsg; - string input_path, labels_path, w1_path, w2_path, wo_path, b1_path, b2_path, - bo_path; - int block_x, block_y, batch_size; - int numFeatures, numNeurons, numLabels; - if (argc < 3) { - cout << "Usage: blockDimensionX blockDimensionY batchSize numFeatures numNeurons numLabels Y" - "path/to/weights/and/bias(leave empty if generate random)" - << endl; - exit(-1); - } - - block_x = atoi(argv[1]); - block_y = atoi(argv[2]); - batch_size = atoi(argv[3]); - numFeatures = atoi(argv[4]); - numNeurons = atoi(argv[5]); - numLabels = atoi(argv[6]); - - if (argc >= 7) { - if (strcmp(argv[7], "N")==0) { - reloadData = false; - std::cout << "WARNING: we do not reload data!" << std::endl; - } - } - cout << "Using block dimensions " << block_x << ", " << block_y << endl; - - bool generate = true; - - string masterIp = "localhost"; - pdb::PDBLoggerPtr clientLogger = make_shared("FFclientLog"); - pdb::PDBClient pdbClient(8108, masterIp, clientLogger, false, true); - pdb::CatalogClient catalogClient(8108, masterIp, clientLogger); - - - - if (reloadData) { - - ff::createDatabase(pdbClient, "ff"); - ff::setup(pdbClient, "ff"); - - ff::createSet(pdbClient, "ff", "inputs", "inputs", 64); - ff::createSet(pdbClient, "ff", "label", "label", 64); - - ff::createSet(pdbClient, "ff", "w1", "W1", 64); - ff::createSet(pdbClient, "ff", "b1", "B1", 64); - - ff::createSet(pdbClient, "ff", "wo", "WO", 64); - ff::createSet(pdbClient, "ff", "bo", "BO", 64); - - } - - ff::createSet(pdbClient, "ff", "output", "Output", 256); - - ff::createSet(pdbClient, "ff", "y1", "Y1", 64); - - - ff::createSet(pdbClient, "ff", "yo", "YO", 64); - - if (!generate && reloadData) { - input_path = string(argv[4]) + "/input.out"; - labels_path = string(argv[4]) + "/labels.out"; - w1_path = string(argv[4]) + "/w1.out"; - wo_path = string(argv[4]) + "/wo.out"; - b1_path = string(argv[4]) + "/b1.out"; - bo_path = string(argv[4]) + "/bo.out"; - - // load the input data - ff::load_matrix_data(pdbClient, input_path, "ff", "inputs", - block_x, block_y, false, false, errMsg); - (void)ff::load_matrix_data(pdbClient, w1_path, "ff", "w1", block_x, block_y, - false, false, errMsg); - (void)ff::load_matrix_data(pdbClient, wo_path, "ff", "wo", block_x, block_y, - false, false, errMsg); - (void)ff::load_matrix_data(pdbClient, b1_path, "ff", "b1", block_x, block_y, - false, true, errMsg); - (void)ff::load_matrix_data(pdbClient, bo_path, "ff", "bo", block_x, block_y, - false, true, errMsg); - } else if (reloadData) { - - std::cout << "To load matrix for ff:inputs" << std::endl; - ff::loadMatrix(pdbClient, "ff", "inputs", batch_size, numFeatures, block_x, - block_y, false, false, errMsg); - - std::cout << "To load matrix for ff:w1" << std::endl; - ff::loadMatrix(pdbClient, "ff", "w1", numNeurons, numFeatures, block_x, block_y, - false, false, errMsg); - - std::cout << "To load matrix for ff:b1" << std::endl; - ff::loadMatrix(pdbClient, "ff", "b1", numNeurons, 1, block_x, - block_y, false, true, errMsg); - - std::cout << "To load matrix for ff:wo" << std::endl; - ff::loadMatrix(pdbClient, "ff", "wo", numLabels, numNeurons, block_x, - block_y, false, false, errMsg); - // 2 x 1 - std::cout << "To load matrix for ff:bo" << std::endl; - ff::loadMatrix(pdbClient, "ff", "bo", numLabels, 1, block_x, block_y, - false, true, errMsg); - } - - double dropout_rate = 0.5; - - - - auto begin = std::chrono::high_resolution_clock::now(); - - ff::inference_unit(pdbClient, "ff", "w1", "wo", "inputs", "b1", "bo", - "output", dropout_rate); - - auto end = std::chrono::high_resolution_clock::now(); - std::cout << "*****FFTest End-to-End Time Duration: ****" - << std::chrono::duration_cast>(end - begin).count() - << " secs." << std::endl; - - - - vector> labels_test; - - if (!generate) - ff::load_matrix_from_file(labels_path, labels_test); - - int count = 0; - int correct = 0; - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - auto it = pdbClient.getSetIterator("ff", "output"); - - - for (auto r : it) { - - count++; - - double *data = r->getRawDataHandle()->c_ptr(); - int i = 0; - int j = r->getBlockRowIndex() * r->getRowNums(); - while (i < r->getRowNums() * r->getColNums()) { - if (!generate && j >= labels_test.size()) - break; - - cout << data[i] << ", " << data[i + 1] << endl; - - if (!generate) { - int pos1 = data[i] > data[i + 1] ? 0 : 1; - int pos2 = labels_test[j][0] > labels_test[j][1] ? 0 : 1; - - if (pos1 == pos2) - correct++; - } - - i += r->getColNums(); - j++; - } - } - - if (!generate) - cout << "Accuracy: " << correct << "/" << labels_test.size() << std::endl; - } - - std::cout << "count=" << count << std::endl; - - sleep(20); - - return 0; -} +#include +#include +#include +#include +#include +#include + +#include "PDBClient.h" + +#include "FFMatrixBlock.h" +#include "FFMatrixUtil.h" +#include "SimpleFF.h" + +using namespace std; + +int main(int argc, char *argv[]) { + + bool reloadData = true; + string errMsg; + string input_path, labels_path, w1_path, w2_path, wo_path, b1_path, b2_path, + bo_path; + int block_x, block_y, batch_size; + int numFeatures, numNeurons, numLabels; + if (argc < 3) { + cout << "Usage: blockDimensionX blockDimensionY batchSize numFeatures numNeurons numLabels Y" + "path/to/weights/and/bias(leave empty if generate random)" + << endl; + exit(-1); + } + + block_x = atoi(argv[1]); + block_y = atoi(argv[2]); + batch_size = atoi(argv[3]); + numFeatures = atoi(argv[4]); + numNeurons = atoi(argv[5]); + numLabels = atoi(argv[6]); + + if (argc >= 7) { + if (strcmp(argv[7], "N")==0) { + reloadData = false; + std::cout << "WARNING: we do not reload data!" << std::endl; + } + } + cout << "Using block dimensions " << block_x << ", " << block_y << endl; + + bool generate = true; + + string masterIp = "localhost"; + pdb::PDBLoggerPtr clientLogger = make_shared("FFclientLog"); + pdb::PDBClient pdbClient(8108, masterIp, clientLogger, false, true); + pdb::CatalogClient catalogClient(8108, masterIp, clientLogger); + + + + if (reloadData) { + + ff::createDatabase(pdbClient, "ff"); + ff::setup(pdbClient, "ff"); + + ff::createSet(pdbClient, "ff", "inputs", "inputs", 64); + ff::createSet(pdbClient, "ff", "label", "label", 64); + + ff::createSet(pdbClient, "ff", "w1", "W1", 64); + ff::createSet(pdbClient, "ff", "b1", "B1", 64); + + ff::createSet(pdbClient, "ff", "wo", "WO", 64); + ff::createSet(pdbClient, "ff", "bo", "BO", 64); + + } + + ff::createSet(pdbClient, "ff", "output", "Output", 256); + + ff::createSet(pdbClient, "ff", "y1", "Y1", 64); + + + ff::createSet(pdbClient, "ff", "yo", "YO", 64); + + if (!generate && reloadData) { + input_path = string(argv[4]) + "/input.out"; + labels_path = string(argv[4]) + "/labels.out"; + w1_path = string(argv[4]) + "/w1.out"; + wo_path = string(argv[4]) + "/wo.out"; + b1_path = string(argv[4]) + "/b1.out"; + bo_path = string(argv[4]) + "/bo.out"; + + // load the input data + ff::load_matrix_data(pdbClient, input_path, "ff", "inputs", + block_x, block_y, false, false, errMsg); + (void)ff::load_matrix_data(pdbClient, w1_path, "ff", "w1", block_x, block_y, + false, false, errMsg); + (void)ff::load_matrix_data(pdbClient, wo_path, "ff", "wo", block_x, block_y, + false, false, errMsg); + (void)ff::load_matrix_data(pdbClient, b1_path, "ff", "b1", block_x, block_y, + false, true, errMsg); + (void)ff::load_matrix_data(pdbClient, bo_path, "ff", "bo", block_x, block_y, + false, true, errMsg); + } else if (reloadData) { + + std::cout << "To load matrix for ff:inputs" << std::endl; + ff::loadMatrix(pdbClient, "ff", "inputs", batch_size, numFeatures, block_x, + block_y, false, false, errMsg); + + std::cout << "To load matrix for ff:w1" << std::endl; + ff::loadMatrix(pdbClient, "ff", "w1", numNeurons, numFeatures, block_x, block_y, + false, false, errMsg); + + std::cout << "To load matrix for ff:b1" << std::endl; + ff::loadMatrix(pdbClient, "ff", "b1", numNeurons, 1, block_x, + block_y, false, true, errMsg); + + std::cout << "To load matrix for ff:wo" << std::endl; + ff::loadMatrix(pdbClient, "ff", "wo", numLabels, numNeurons, block_x, + block_y, false, false, errMsg); + // 2 x 1 + std::cout << "To load matrix for ff:bo" << std::endl; + ff::loadMatrix(pdbClient, "ff", "bo", numLabels, 1, block_x, block_y, + false, true, errMsg); + } + + double dropout_rate = 0.5; + + + + auto begin = std::chrono::high_resolution_clock::now(); + + ff::inference_unit(pdbClient, "ff", "w1", "wo", "inputs", "b1", "bo", + "output", dropout_rate); + + auto end = std::chrono::high_resolution_clock::now(); + std::cout << "*****FFTest End-to-End Time Duration: ****" + << std::chrono::duration_cast>(end - begin).count() + << " secs." << std::endl; + + + + vector> labels_test; + + if (!generate) + ff::load_matrix_from_file(labels_path, labels_test); + + int count = 0; + int correct = 0; + { + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + auto it = pdbClient.getSetIterator("ff", "output"); + + + for (auto r : it) { + + count++; + + float *data = r->getRawDataHandle()->c_ptr(); + int i = 0; + int j = r->getBlockRowIndex() * r->getRowNums(); + while (i < r->getRowNums() * r->getColNums()) { + if (!generate && j >= labels_test.size()) + break; + + cout << data[i] << ", " << data[i + 1] << endl; + + if (!generate) { + int pos1 = data[i] > data[i + 1] ? 0 : 1; + int pos2 = labels_test[j][0] > labels_test[j][1] ? 0 : 1; + + if (pos1 == pos2) + correct++; + } + + i += r->getColNums(); + j++; + } + } + + if (!generate) + cout << "Accuracy: " << correct << "/" << labels_test.size() << std::endl; + } + + std::cout << "count=" << count << std::endl; + + sleep(20); + + return 0; +} diff --git a/src/tests/source/LSTMTest.cc b/src/tests/source/LSTMTest.cc index 6963c461..8dc70bf5 100644 --- a/src/tests/source/LSTMTest.cc +++ b/src/tests/source/LSTMTest.cc @@ -1,559 +1,559 @@ -#include -#include -#include -#include -#include - -#include "FFMatrixBlock.h" -#include "FFMatrixMeta.h" -#include "FFMatrixData.h" - -#include "FFMatrixBlockScanner.h" -#include "FFMatrixWriter.h" - -#include "FFInputLayerJoin.h" -#include "FFAggMatrix.h" - -#include "LSTMThreeWaySum.h" -#include "LSTMTwoSum.h" -#include "LSTMHiddenState.h" - -#include "PDBClient.h" - -using namespace std; - -void print(pdb::PDBClient &pdbClient, string dbName, string setName) { - auto it = pdbClient.getSetIterator(dbName, setName); - - for (auto r : it) { - double *data = r->getRawDataHandle()->c_ptr(); - for (int i = 0; i < r->getRowNums() * r->getColNums(); i++) { - std::cout << data[i] << ","; - } - } -} - -void print_stats(pdb::PDBClient &pdbClient, string dbName, string setName) { - int rows = 0, cols = 0, blocks = 0; - int totalRows = 0, totalCols = 0; - int blockRows = 0, blockCols = 0; - auto it = pdbClient.getSetIterator(dbName, setName); - - for (auto r : it) { - std::cout << r->getBlockRowIndex() << "," << r->getBlockColIndex() << ";"; - rows = r->getRowNums(); - cols = r->getColNums(); - if (r->getBlockRowIndex() == 0) { - totalRows += r->getRowNums(); - blockRows += 1; - } - if (r->getBlockColIndex() == 0) { - totalCols += r->getColNums(); - blockCols += 1; - } - blocks++; - } - - std::cout << "\n" - << setName << " (" << blockRows << " X " << blockCols << ") (" - << blocks << ") : (" << totalRows << " x " << totalCols - << "), Each block size: " << rows << " x " << cols << std::endl; -} - -void loadMatrix(pdb::PDBClient &pdbClient, String dbName, String setName, - int totalX, int totalY, int blockX, int blockY, int initVal, - std::string &errMsg) { - - int total = 0; - pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); - - pdb::Handle>> storeMatrix1 = - pdb::makeObject>>(); - - int numXBlocks = ceil(totalX / (double)blockX); - int numYBlocks = ceil(totalY / (double)blockY); - - try { - for (int i = 0; i < numXBlocks; i++) { - for (int j = 0; j < numYBlocks; j++) { - pdb::Handle myData = - pdb::makeObject(i, j, blockX, blockY); - - for (int ii = 0; ii < blockX; ii++) { - for (int jj = 0; jj < blockY; jj++) { - // row = i * blockX + ii, col = j * blockY + jj - double data = - (i * blockX + ii) >= totalX || (j * blockY + jj) >= totalY - ? 0 - : initVal == -1 ? i + j + ii + jj : initVal; - (*(myData->getRawDataHandle()))[ii * blockY + jj] = data; - } - } - - // std::cout << "New block: " << total << std::endl; - storeMatrix1->push_back(myData); - total++; - } - } - if (!pdbClient.sendData( - std::pair(setName, dbName), storeMatrix1, - errMsg)) { - std::cout << "Failed to send data to dispatcher server" << std::endl; - exit(1); - } - } catch (NotEnoughSpace &e) { - std::cout << "Failed to send data to dispatcher server" << std::endl; - exit(1); - } - std::cout << setName << "(" << totalX << "x" << totalY << "): (" << numXBlocks - << " x " << numYBlocks << ")" << total << " blocks = " << blockX - << " x " << blockY << " each" << std::endl; - - // to write back all buffered records - pdbClient.flushData(errMsg); -} - -void loadLibrary(pdb::PDBClient &pdbClient, string path) { - string errMsg; - if (!pdbClient.registerType(path, errMsg)) { - cout << "Couldnt include " << path << ": " << errMsg << endl; - exit(-1); - } -} - -void createSet(pdb::PDBClient &pdbClient, string dbName, string setName, - string setName1) { - string errMsg; - if (!pdbClient.createSet( - dbName, setName, errMsg, (size_t)64 * (size_t)1024 * (size_t)1024, - setName1)) { - cout << "Not able to create set: " + errMsg; - exit(-1); - } else { - cout << "Created set.\n"; - } -} - -void createDatabase(pdb::PDBClient &pdbClient, string dbName) { - string errMsg; - if (!pdbClient.createDatabase(dbName, errMsg)) { - cout << "Not able to create database: " << errMsg << endl; - exit(-1); - } else { - cout << "Created database" << endl; - } -} - -int main(int argc, char *argv[]) { - string errMsg; - - string masterIp = "localhost"; - pdb::PDBLoggerPtr clientLogger = make_shared("LSTMclientLog"); - pdb::PDBClient pdbClient(8108, masterIp, clientLogger, false, true); - pdb::CatalogClient catalogClient(8108, masterIp, clientLogger); - - loadLibrary(pdbClient, "libraries/libFFMatrixMeta.so"); - loadLibrary(pdbClient, "libraries/libFFMatrixData.so"); - loadLibrary(pdbClient, "libraries/libFFMatrixBlock.so"); - - loadLibrary(pdbClient, "libraries/libFFMatrixBlockScanner.so"); - loadLibrary(pdbClient, "libraries/libFFMatrixWriter.so"); - - loadLibrary(pdbClient, "libraries/libFFInputLayerJoin.so"); - loadLibrary(pdbClient, "libraries/libFFAggMatrix.so"); - - loadLibrary(pdbClient, "libraries/libLSTMThreeWaySum.so"); - loadLibrary(pdbClient, "libraries/libLSTMTwoSum.so"); - loadLibrary(pdbClient, "libraries/libLSTMHiddenState.so"); - - - createDatabase(pdbClient, "lstm"); - - createSet(pdbClient, "lstm", "f_t", "Forget"); - createSet(pdbClient, "lstm", "x_t", "Input"); - createSet(pdbClient, "lstm", "o_t", "Output"); - createSet(pdbClient, "lstm", "c_t_1", "CellState"); - createSet(pdbClient, "lstm", "c_t_temp", "CellStateTemp"); - createSet(pdbClient, "lstm", "h_t_1", "HiddenState"); - - createSet(pdbClient, "lstm", "w_f", "WForget"); - createSet(pdbClient, "lstm", "w_i", "WInput"); - createSet(pdbClient, "lstm", "w_o", "WOutput"); - createSet(pdbClient, "lstm", "w_c", "WCellState"); - - createSet(pdbClient, "lstm", "u_f", "UForget"); - createSet(pdbClient, "lstm", "u_i", "UInput"); - createSet(pdbClient, "lstm", "u_o", "UOutput"); - createSet(pdbClient, "lstm", "u_c", "UCellState"); - - createSet(pdbClient, "lstm", "b_f", "BForget"); - createSet(pdbClient, "lstm", "b_i", "BInput"); - createSet(pdbClient, "lstm", "b_o", "BOutput"); - createSet(pdbClient, "lstm", "b_c", "BCellState"); - - createSet(pdbClient, "lstm", "c_t", "CellStateNext"); - createSet(pdbClient, "lstm", "h_t", "HiddenStateNext"); - createSet(pdbClient, "lstm", "i_t", "InputGate"); - - int D = 400; // features - int B = 500; // batch size - int L = 128; // output labels? - int block_x = 100; - int block_y = 100; - - loadMatrix(pdbClient, "lstm", "w_f", L, D, block_x, block_y, 0, errMsg); - loadMatrix(pdbClient, "lstm", "x_t", D, B, block_x, block_y, -1, errMsg); - loadMatrix(pdbClient, "lstm", "u_f", L, L, block_x, block_y, 0, errMsg); - loadMatrix(pdbClient, "lstm", "b_f", L, B, block_x, block_y, 0, errMsg); - loadMatrix(pdbClient, "lstm", "h_t_1", L, B, block_x, block_y, 0, errMsg); - - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - pdb::Handle w_f = - pdb::makeObject("lstm", "w_f"); - pdb::Handle x_t = - pdb::makeObject("lstm", "x_t"); - - // multiply - Handle w_forget_x_join = makeObject(); - w_forget_x_join->setInput(0, w_f); - w_forget_x_join->setInput(1, x_t); - - Handle w_forget_x_agg = makeObject(); - w_forget_x_agg->setInput(w_forget_x_join); - - - pdb::Handle u_f = - pdb::makeObject("lstm", "u_f"); - pdb::Handle h_t_1 = - pdb::makeObject("lstm", "h_t_1"); - - Handle u_forget_h_join = makeObject(); - u_forget_h_join->setInput(0, u_f); - u_forget_h_join->setInput(1, h_t_1); - - Handle u_forget_h_agg = makeObject(); - u_forget_h_agg->setInput(u_forget_h_join); - - - pdb::Handle b_f = - pdb::makeObject("lstm", "b_f"); - - // add and sigmod - pdb::Handle sum = pdb::makeObject(); - sum->setInput(0, w_forget_x_agg); - sum->setInput(1, u_forget_h_agg); - sum->setInput(2, b_f); - - // make the writer - pdb::Handle myWriter = pdb::makeObject("lstm", "f_t"); - myWriter->setInput(sum); - - // run the computation - if (!pdbClient.executeComputations(errMsg, myWriter)) { - std::cout << "Computation failed. Message was: " << errMsg << "\n"; - return 1; - } - } - - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - print_stats(pdbClient, "lstm", "w_f"); - print_stats(pdbClient, "lstm", "x_t"); - print_stats(pdbClient, "lstm", "u_f"); - print_stats(pdbClient, "lstm", "h_t_1"); - print_stats(pdbClient, "lstm", "b_f"); - print_stats(pdbClient, "lstm", "f_t"); - } - - loadMatrix(pdbClient, "lstm", "w_i", L, D, block_x, block_y, 0, errMsg); - loadMatrix(pdbClient, "lstm", "u_i", L, L, block_x, block_y, 0, errMsg); - loadMatrix(pdbClient, "lstm", "b_i", L, B, block_x, block_y, 0, errMsg); - - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - pdb::Handle w_i = - pdb::makeObject("lstm", "w_i"); - pdb::Handle x_t = - pdb::makeObject("lstm", "x_t"); - - // multiply - Handle w_input_x_join = makeObject(); - w_input_x_join->setInput(0, w_i); - w_input_x_join->setInput(1, x_t); - - Handle w_input_x_agg = makeObject(); - w_input_x_agg->setInput(w_input_x_join); - - - pdb::Handle u_i = - pdb::makeObject("lstm", "u_i"); - pdb::Handle h_t_1 = - pdb::makeObject("lstm", "h_t_1"); - - Handle u_input_h_join = makeObject(); - u_input_h_join->setInput(0, u_i); - u_input_h_join->setInput(1, h_t_1); - - Handle u_input_h_agg = makeObject(); - u_input_h_agg->setInput(u_input_h_join); - - - pdb::Handle b_i = - pdb::makeObject("lstm", "b_i"); - - // add and sigmod - pdb::Handle sum = pdb::makeObject(); - sum->setInput(0, w_input_x_agg); - sum->setInput(1, u_input_h_agg); - sum->setInput(2, b_i); - - // make the writer - pdb::Handle myWriter = pdb::makeObject("lstm", "i_t"); - myWriter->setInput(sum); - - // run the computation - if (!pdbClient.executeComputations(errMsg, myWriter)) { - std::cout << "Computation failed. Message was: " << errMsg << "\n"; - return 1; - } - } - - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - print_stats(pdbClient, "lstm", "w_i"); - print_stats(pdbClient, "lstm", "x_t"); - print_stats(pdbClient, "lstm", "u_i"); - print_stats(pdbClient, "lstm", "h_t_1"); - print_stats(pdbClient, "lstm", "b_i"); - print_stats(pdbClient, "lstm", "i_t"); - } - - loadMatrix(pdbClient, "lstm", "w_o", L, D, block_x, block_y, 0, errMsg); - loadMatrix(pdbClient, "lstm", "u_o", L, L, block_x, block_y, 0, errMsg); - loadMatrix(pdbClient, "lstm", "b_o", L, B, block_x, block_y, 0, errMsg); - - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - pdb::Handle w_o = - pdb::makeObject("lstm", "w_o"); - pdb::Handle x_t = - pdb::makeObject("lstm", "x_t"); - - // multiply - Handle w_output_x_join = makeObject(); - w_output_x_join->setInput(0, w_o); - w_output_x_join->setInput(1, x_t); - - Handle w_output_x_agg = makeObject(); - w_output_x_agg->setInput(w_output_x_join); - - - pdb::Handle u_o = - pdb::makeObject("lstm", "u_o"); - pdb::Handle h_t_1 = - pdb::makeObject("lstm", "h_t_1"); - - Handle u_output_h_join = makeObject(); - u_output_h_join->setInput(0, u_o); - u_output_h_join->setInput(1, h_t_1); - - Handle u_output_h_agg = makeObject(); - u_output_h_agg->setInput(u_output_h_join); - - - pdb::Handle b_o = - pdb::makeObject("lstm", "b_o"); - - // add and sigmod - pdb::Handle sum = pdb::makeObject(); - sum->setInput(0, w_output_x_agg); - sum->setInput(1, u_output_h_agg); - sum->setInput(2, b_o); - - // make the writer - pdb::Handle myWriter = pdb::makeObject("lstm", "o_t"); - myWriter->setInput(sum); - - // run the computation - if (!pdbClient.executeComputations(errMsg, myWriter)) { - std::cout << "Computation failed. Message was: " << errMsg << "\n"; - return 1; - } - } - - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - print_stats(pdbClient, "lstm", "w_o"); - print_stats(pdbClient, "lstm", "x_t"); - print_stats(pdbClient, "lstm", "u_o"); - print_stats(pdbClient, "lstm", "h_t_1"); - print_stats(pdbClient, "lstm", "b_o"); - print_stats(pdbClient, "lstm", "o_t"); - } - - loadMatrix(pdbClient, "lstm", "w_c", L, D, block_x, block_y, 0, errMsg); - loadMatrix(pdbClient, "lstm", "u_c", L, L, block_x, block_y, 0, errMsg); - loadMatrix(pdbClient, "lstm", "b_c", L, B, block_x, block_y, 0, errMsg); - - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - pdb::Handle w_c = - pdb::makeObject("lstm", "w_c"); - pdb::Handle x_t = - pdb::makeObject("lstm", "x_t"); - - // multiply - Handle w_cell_state_join = makeObject(); - w_cell_state_join->setInput(0, w_c); - w_cell_state_join->setInput(1, x_t); - - Handle w_cell_state_agg = makeObject(); - w_cell_state_agg->setInput(w_cell_state_join); - - - pdb::Handle u_c = - pdb::makeObject("lstm", "u_c"); - pdb::Handle h_t_1 = - pdb::makeObject("lstm", "h_t_1"); - - Handle u_cell_state_h_join = makeObject(); - u_cell_state_h_join->setInput(0, u_c); - u_cell_state_h_join->setInput(1, h_t_1); - - Handle u_cell_state_h_agg = makeObject(); - u_cell_state_h_agg->setInput(u_cell_state_h_join); - - - pdb::Handle b_c = - pdb::makeObject("lstm", "b_c"); - - // add and tanh - pdb::Handle sum = pdb::makeObject(SumActivation::Tanh); - sum->setInput(0, w_cell_state_agg); - sum->setInput(1, u_cell_state_h_agg); - sum->setInput(2, b_c); - - // make the writer - pdb::Handle myWriter = pdb::makeObject("lstm", "c_t_temp"); - myWriter->setInput(sum); - - // run the computation - if (!pdbClient.executeComputations(errMsg, myWriter)) { - std::cout << "Computation failed. Message was: " << errMsg << "\n"; - return 1; - } - } - - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - print_stats(pdbClient, "lstm", "w_c"); - print_stats(pdbClient, "lstm", "x_t"); - print_stats(pdbClient, "lstm", "u_c"); - print_stats(pdbClient, "lstm", "h_t_1"); - print_stats(pdbClient, "lstm", "b_c"); - print_stats(pdbClient, "lstm", "c_t_temp"); - } - - loadMatrix(pdbClient, "lstm", "c_t_1", L, B, block_x, block_y, 0, errMsg); - - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - pdb::Handle f_t = - pdb::makeObject("lstm", "f_t"); - pdb::Handle c_t_1 = - pdb::makeObject("lstm", "c_t_1"); - - // multiply - Handle forget_cell_state_join = makeObject(); - forget_cell_state_join->setInput(0, f_t); - forget_cell_state_join->setInput(1, c_t_1); - - Handle forget_cell_state_agg = makeObject(); - forget_cell_state_agg->setInput(forget_cell_state_join); - - pdb::Handle i_t = - pdb::makeObject("lstm", "i_t"); - pdb::Handle c_t_temp = - pdb::makeObject("lstm", "c_t_temp"); - - Handle input_cell_state_temp_join = makeObject(); - input_cell_state_temp_join->setInput(0, i_t); - input_cell_state_temp_join->setInput(1, c_t_temp); - - Handle input_cell_state_temp_agg = makeObject(); - input_cell_state_temp_agg->setInput(input_cell_state_temp_join); - - // add - pdb::Handle sum = pdb::makeObject(); - sum->setInput(0, forget_cell_state_agg); - sum->setInput(1, input_cell_state_temp_agg); - - // make the writer - pdb::Handle myWriter = pdb::makeObject("lstm", "c_t"); - myWriter->setInput(sum); - - // run the computation - if (!pdbClient.executeComputations(errMsg, myWriter)) { - std::cout << "Computation failed. Message was: " << errMsg << "\n"; - return 1; - } - } - - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - print_stats(pdbClient, "lstm", "f_t"); - print_stats(pdbClient, "lstm", "c_t_1"); - print_stats(pdbClient, "lstm", "i_t"); - print_stats(pdbClient, "lstm", "c_t_temp"); - print_stats(pdbClient, "lstm", "c_t"); - } - - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - pdb::Handle o_t = - pdb::makeObject("lstm", "o_t"); - pdb::Handle c_t_1 = - pdb::makeObject("lstm", "c_t"); - - // multiply - Handle hidden_state_join = makeObject(); - hidden_state_join->setInput(0, o_t); - hidden_state_join->setInput(1, c_t_1); - - Handle hidden_state_agg = makeObject(); - hidden_state_agg->setInput(hidden_state_join); - - // make the writer - pdb::Handle myWriter = pdb::makeObject("lstm", "h_t"); - myWriter->setInput(hidden_state_agg); - - // run the computation - if (!pdbClient.executeComputations(errMsg, myWriter)) { - std::cout << "Computation failed. Message was: " << errMsg << "\n"; - return 1; - } - } - - { - const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; - - print_stats(pdbClient, "lstm", "o_t"); - print_stats(pdbClient, "lstm", "c_t"); - print_stats(pdbClient, "lstm", "h_t"); - } - - - return 0; -} +#include +#include +#include +#include +#include + +#include "FFMatrixBlock.h" +#include "FFMatrixMeta.h" +#include "FFMatrixData.h" + +#include "FFMatrixBlockScanner.h" +#include "FFMatrixWriter.h" + +#include "FFInputLayerJoin.h" +#include "FFAggMatrix.h" + +#include "LSTMThreeWaySum.h" +#include "LSTMTwoSum.h" +#include "LSTMHiddenState.h" + +#include "PDBClient.h" + +using namespace std; + +void print(pdb::PDBClient &pdbClient, string dbName, string setName) { + auto it = pdbClient.getSetIterator(dbName, setName); + + for (auto r : it) { + float *data = r->getRawDataHandle()->c_ptr(); + for (int i = 0; i < r->getRowNums() * r->getColNums(); i++) { + std::cout << data[i] << ","; + } + } +} + +void print_stats(pdb::PDBClient &pdbClient, string dbName, string setName) { + int rows = 0, cols = 0, blocks = 0; + int totalRows = 0, totalCols = 0; + int blockRows = 0, blockCols = 0; + auto it = pdbClient.getSetIterator(dbName, setName); + + for (auto r : it) { + std::cout << r->getBlockRowIndex() << "," << r->getBlockColIndex() << ";"; + rows = r->getRowNums(); + cols = r->getColNums(); + if (r->getBlockRowIndex() == 0) { + totalRows += r->getRowNums(); + blockRows += 1; + } + if (r->getBlockColIndex() == 0) { + totalCols += r->getColNums(); + blockCols += 1; + } + blocks++; + } + + std::cout << "\n" + << setName << " (" << blockRows << " X " << blockCols << ") (" + << blocks << ") : (" << totalRows << " x " << totalCols + << "), Each block size: " << rows << " x " << cols << std::endl; +} + +void loadMatrix(pdb::PDBClient &pdbClient, String dbName, String setName, + int totalX, int totalY, int blockX, int blockY, int initVal, + std::string &errMsg) { + + int total = 0; + pdb::makeObjectAllocatorBlock(128 * 1024 * 1024, true); + + pdb::Handle>> storeMatrix1 = + pdb::makeObject>>(); + + int numXBlocks = ceil(totalX / (double)blockX); + int numYBlocks = ceil(totalY / (double)blockY); + + try { + for (int i = 0; i < numXBlocks; i++) { + for (int j = 0; j < numYBlocks; j++) { + pdb::Handle myData = + pdb::makeObject(i, j, blockX, blockY); + + for (int ii = 0; ii < blockX; ii++) { + for (int jj = 0; jj < blockY; jj++) { + // row = i * blockX + ii, col = j * blockY + jj + double data = + (i * blockX + ii) >= totalX || (j * blockY + jj) >= totalY + ? 0 + : initVal == -1 ? i + j + ii + jj : initVal; + (*(myData->getRawDataHandle()))[ii * blockY + jj] = data; + } + } + + // std::cout << "New block: " << total << std::endl; + storeMatrix1->push_back(myData); + total++; + } + } + if (!pdbClient.sendData( + std::pair(setName, dbName), storeMatrix1, + errMsg)) { + std::cout << "Failed to send data to dispatcher server" << std::endl; + exit(1); + } + } catch (NotEnoughSpace &e) { + std::cout << "Failed to send data to dispatcher server" << std::endl; + exit(1); + } + std::cout << setName << "(" << totalX << "x" << totalY << "): (" << numXBlocks + << " x " << numYBlocks << ")" << total << " blocks = " << blockX + << " x " << blockY << " each" << std::endl; + + // to write back all buffered records + pdbClient.flushData(errMsg); +} + +void loadLibrary(pdb::PDBClient &pdbClient, string path) { + string errMsg; + if (!pdbClient.registerType(path, errMsg)) { + cout << "Couldnt include " << path << ": " << errMsg << endl; + exit(-1); + } +} + +void createSet(pdb::PDBClient &pdbClient, string dbName, string setName, + string setName1) { + string errMsg; + if (!pdbClient.createSet( + dbName, setName, errMsg, (size_t)64 * (size_t)1024 * (size_t)1024, + setName1)) { + cout << "Not able to create set: " + errMsg; + exit(-1); + } else { + cout << "Created set.\n"; + } +} + +void createDatabase(pdb::PDBClient &pdbClient, string dbName) { + string errMsg; + if (!pdbClient.createDatabase(dbName, errMsg)) { + cout << "Not able to create database: " << errMsg << endl; + exit(-1); + } else { + cout << "Created database" << endl; + } +} + +int main(int argc, char *argv[]) { + string errMsg; + + string masterIp = "localhost"; + pdb::PDBLoggerPtr clientLogger = make_shared("LSTMclientLog"); + pdb::PDBClient pdbClient(8108, masterIp, clientLogger, false, true); + pdb::CatalogClient catalogClient(8108, masterIp, clientLogger); + + loadLibrary(pdbClient, "libraries/libFFMatrixMeta.so"); + loadLibrary(pdbClient, "libraries/libFFMatrixData.so"); + loadLibrary(pdbClient, "libraries/libFFMatrixBlock.so"); + + loadLibrary(pdbClient, "libraries/libFFMatrixBlockScanner.so"); + loadLibrary(pdbClient, "libraries/libFFMatrixWriter.so"); + + loadLibrary(pdbClient, "libraries/libFFInputLayerJoin.so"); + loadLibrary(pdbClient, "libraries/libFFAggMatrix.so"); + + loadLibrary(pdbClient, "libraries/libLSTMThreeWaySum.so"); + loadLibrary(pdbClient, "libraries/libLSTMTwoSum.so"); + loadLibrary(pdbClient, "libraries/libLSTMHiddenState.so"); + + + createDatabase(pdbClient, "lstm"); + + createSet(pdbClient, "lstm", "f_t", "Forget"); + createSet(pdbClient, "lstm", "x_t", "Input"); + createSet(pdbClient, "lstm", "o_t", "Output"); + createSet(pdbClient, "lstm", "c_t_1", "CellState"); + createSet(pdbClient, "lstm", "c_t_temp", "CellStateTemp"); + createSet(pdbClient, "lstm", "h_t_1", "HiddenState"); + + createSet(pdbClient, "lstm", "w_f", "WForget"); + createSet(pdbClient, "lstm", "w_i", "WInput"); + createSet(pdbClient, "lstm", "w_o", "WOutput"); + createSet(pdbClient, "lstm", "w_c", "WCellState"); + + createSet(pdbClient, "lstm", "u_f", "UForget"); + createSet(pdbClient, "lstm", "u_i", "UInput"); + createSet(pdbClient, "lstm", "u_o", "UOutput"); + createSet(pdbClient, "lstm", "u_c", "UCellState"); + + createSet(pdbClient, "lstm", "b_f", "BForget"); + createSet(pdbClient, "lstm", "b_i", "BInput"); + createSet(pdbClient, "lstm", "b_o", "BOutput"); + createSet(pdbClient, "lstm", "b_c", "BCellState"); + + createSet(pdbClient, "lstm", "c_t", "CellStateNext"); + createSet(pdbClient, "lstm", "h_t", "HiddenStateNext"); + createSet(pdbClient, "lstm", "i_t", "InputGate"); + + int D = 400; // features + int B = 500; // batch size + int L = 128; // output labels? + int block_x = 100; + int block_y = 100; + + loadMatrix(pdbClient, "lstm", "w_f", L, D, block_x, block_y, 0, errMsg); + loadMatrix(pdbClient, "lstm", "x_t", D, B, block_x, block_y, -1, errMsg); + loadMatrix(pdbClient, "lstm", "u_f", L, L, block_x, block_y, 0, errMsg); + loadMatrix(pdbClient, "lstm", "b_f", L, B, block_x, block_y, 0, errMsg); + loadMatrix(pdbClient, "lstm", "h_t_1", L, B, block_x, block_y, 0, errMsg); + + { + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + pdb::Handle w_f = + pdb::makeObject("lstm", "w_f"); + pdb::Handle x_t = + pdb::makeObject("lstm", "x_t"); + + // multiply + Handle w_forget_x_join = makeObject(); + w_forget_x_join->setInput(0, w_f); + w_forget_x_join->setInput(1, x_t); + + Handle w_forget_x_agg = makeObject(); + w_forget_x_agg->setInput(w_forget_x_join); + + + pdb::Handle u_f = + pdb::makeObject("lstm", "u_f"); + pdb::Handle h_t_1 = + pdb::makeObject("lstm", "h_t_1"); + + Handle u_forget_h_join = makeObject(); + u_forget_h_join->setInput(0, u_f); + u_forget_h_join->setInput(1, h_t_1); + + Handle u_forget_h_agg = makeObject(); + u_forget_h_agg->setInput(u_forget_h_join); + + + pdb::Handle b_f = + pdb::makeObject("lstm", "b_f"); + + // add and sigmod + pdb::Handle sum = pdb::makeObject(); + sum->setInput(0, w_forget_x_agg); + sum->setInput(1, u_forget_h_agg); + sum->setInput(2, b_f); + + // make the writer + pdb::Handle myWriter = pdb::makeObject("lstm", "f_t"); + myWriter->setInput(sum); + + // run the computation + if (!pdbClient.executeComputations(errMsg, myWriter)) { + std::cout << "Computation failed. Message was: " << errMsg << "\n"; + return 1; + } + } + + { + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + print_stats(pdbClient, "lstm", "w_f"); + print_stats(pdbClient, "lstm", "x_t"); + print_stats(pdbClient, "lstm", "u_f"); + print_stats(pdbClient, "lstm", "h_t_1"); + print_stats(pdbClient, "lstm", "b_f"); + print_stats(pdbClient, "lstm", "f_t"); + } + + loadMatrix(pdbClient, "lstm", "w_i", L, D, block_x, block_y, 0, errMsg); + loadMatrix(pdbClient, "lstm", "u_i", L, L, block_x, block_y, 0, errMsg); + loadMatrix(pdbClient, "lstm", "b_i", L, B, block_x, block_y, 0, errMsg); + + { + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + pdb::Handle w_i = + pdb::makeObject("lstm", "w_i"); + pdb::Handle x_t = + pdb::makeObject("lstm", "x_t"); + + // multiply + Handle w_input_x_join = makeObject(); + w_input_x_join->setInput(0, w_i); + w_input_x_join->setInput(1, x_t); + + Handle w_input_x_agg = makeObject(); + w_input_x_agg->setInput(w_input_x_join); + + + pdb::Handle u_i = + pdb::makeObject("lstm", "u_i"); + pdb::Handle h_t_1 = + pdb::makeObject("lstm", "h_t_1"); + + Handle u_input_h_join = makeObject(); + u_input_h_join->setInput(0, u_i); + u_input_h_join->setInput(1, h_t_1); + + Handle u_input_h_agg = makeObject(); + u_input_h_agg->setInput(u_input_h_join); + + + pdb::Handle b_i = + pdb::makeObject("lstm", "b_i"); + + // add and sigmod + pdb::Handle sum = pdb::makeObject(); + sum->setInput(0, w_input_x_agg); + sum->setInput(1, u_input_h_agg); + sum->setInput(2, b_i); + + // make the writer + pdb::Handle myWriter = pdb::makeObject("lstm", "i_t"); + myWriter->setInput(sum); + + // run the computation + if (!pdbClient.executeComputations(errMsg, myWriter)) { + std::cout << "Computation failed. Message was: " << errMsg << "\n"; + return 1; + } + } + + { + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + print_stats(pdbClient, "lstm", "w_i"); + print_stats(pdbClient, "lstm", "x_t"); + print_stats(pdbClient, "lstm", "u_i"); + print_stats(pdbClient, "lstm", "h_t_1"); + print_stats(pdbClient, "lstm", "b_i"); + print_stats(pdbClient, "lstm", "i_t"); + } + + loadMatrix(pdbClient, "lstm", "w_o", L, D, block_x, block_y, 0, errMsg); + loadMatrix(pdbClient, "lstm", "u_o", L, L, block_x, block_y, 0, errMsg); + loadMatrix(pdbClient, "lstm", "b_o", L, B, block_x, block_y, 0, errMsg); + + { + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + pdb::Handle w_o = + pdb::makeObject("lstm", "w_o"); + pdb::Handle x_t = + pdb::makeObject("lstm", "x_t"); + + // multiply + Handle w_output_x_join = makeObject(); + w_output_x_join->setInput(0, w_o); + w_output_x_join->setInput(1, x_t); + + Handle w_output_x_agg = makeObject(); + w_output_x_agg->setInput(w_output_x_join); + + + pdb::Handle u_o = + pdb::makeObject("lstm", "u_o"); + pdb::Handle h_t_1 = + pdb::makeObject("lstm", "h_t_1"); + + Handle u_output_h_join = makeObject(); + u_output_h_join->setInput(0, u_o); + u_output_h_join->setInput(1, h_t_1); + + Handle u_output_h_agg = makeObject(); + u_output_h_agg->setInput(u_output_h_join); + + + pdb::Handle b_o = + pdb::makeObject("lstm", "b_o"); + + // add and sigmod + pdb::Handle sum = pdb::makeObject(); + sum->setInput(0, w_output_x_agg); + sum->setInput(1, u_output_h_agg); + sum->setInput(2, b_o); + + // make the writer + pdb::Handle myWriter = pdb::makeObject("lstm", "o_t"); + myWriter->setInput(sum); + + // run the computation + if (!pdbClient.executeComputations(errMsg, myWriter)) { + std::cout << "Computation failed. Message was: " << errMsg << "\n"; + return 1; + } + } + + { + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + print_stats(pdbClient, "lstm", "w_o"); + print_stats(pdbClient, "lstm", "x_t"); + print_stats(pdbClient, "lstm", "u_o"); + print_stats(pdbClient, "lstm", "h_t_1"); + print_stats(pdbClient, "lstm", "b_o"); + print_stats(pdbClient, "lstm", "o_t"); + } + + loadMatrix(pdbClient, "lstm", "w_c", L, D, block_x, block_y, 0, errMsg); + loadMatrix(pdbClient, "lstm", "u_c", L, L, block_x, block_y, 0, errMsg); + loadMatrix(pdbClient, "lstm", "b_c", L, B, block_x, block_y, 0, errMsg); + + { + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + pdb::Handle w_c = + pdb::makeObject("lstm", "w_c"); + pdb::Handle x_t = + pdb::makeObject("lstm", "x_t"); + + // multiply + Handle w_cell_state_join = makeObject(); + w_cell_state_join->setInput(0, w_c); + w_cell_state_join->setInput(1, x_t); + + Handle w_cell_state_agg = makeObject(); + w_cell_state_agg->setInput(w_cell_state_join); + + + pdb::Handle u_c = + pdb::makeObject("lstm", "u_c"); + pdb::Handle h_t_1 = + pdb::makeObject("lstm", "h_t_1"); + + Handle u_cell_state_h_join = makeObject(); + u_cell_state_h_join->setInput(0, u_c); + u_cell_state_h_join->setInput(1, h_t_1); + + Handle u_cell_state_h_agg = makeObject(); + u_cell_state_h_agg->setInput(u_cell_state_h_join); + + + pdb::Handle b_c = + pdb::makeObject("lstm", "b_c"); + + // add and tanh + pdb::Handle sum = pdb::makeObject(SumActivation::Tanh); + sum->setInput(0, w_cell_state_agg); + sum->setInput(1, u_cell_state_h_agg); + sum->setInput(2, b_c); + + // make the writer + pdb::Handle myWriter = pdb::makeObject("lstm", "c_t_temp"); + myWriter->setInput(sum); + + // run the computation + if (!pdbClient.executeComputations(errMsg, myWriter)) { + std::cout << "Computation failed. Message was: " << errMsg << "\n"; + return 1; + } + } + + { + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + print_stats(pdbClient, "lstm", "w_c"); + print_stats(pdbClient, "lstm", "x_t"); + print_stats(pdbClient, "lstm", "u_c"); + print_stats(pdbClient, "lstm", "h_t_1"); + print_stats(pdbClient, "lstm", "b_c"); + print_stats(pdbClient, "lstm", "c_t_temp"); + } + + loadMatrix(pdbClient, "lstm", "c_t_1", L, B, block_x, block_y, 0, errMsg); + + { + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + pdb::Handle f_t = + pdb::makeObject("lstm", "f_t"); + pdb::Handle c_t_1 = + pdb::makeObject("lstm", "c_t_1"); + + // multiply + Handle forget_cell_state_join = makeObject(); + forget_cell_state_join->setInput(0, f_t); + forget_cell_state_join->setInput(1, c_t_1); + + Handle forget_cell_state_agg = makeObject(); + forget_cell_state_agg->setInput(forget_cell_state_join); + + pdb::Handle i_t = + pdb::makeObject("lstm", "i_t"); + pdb::Handle c_t_temp = + pdb::makeObject("lstm", "c_t_temp"); + + Handle input_cell_state_temp_join = makeObject(); + input_cell_state_temp_join->setInput(0, i_t); + input_cell_state_temp_join->setInput(1, c_t_temp); + + Handle input_cell_state_temp_agg = makeObject(); + input_cell_state_temp_agg->setInput(input_cell_state_temp_join); + + // add + pdb::Handle sum = pdb::makeObject(); + sum->setInput(0, forget_cell_state_agg); + sum->setInput(1, input_cell_state_temp_agg); + + // make the writer + pdb::Handle myWriter = pdb::makeObject("lstm", "c_t"); + myWriter->setInput(sum); + + // run the computation + if (!pdbClient.executeComputations(errMsg, myWriter)) { + std::cout << "Computation failed. Message was: " << errMsg << "\n"; + return 1; + } + } + + { + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + print_stats(pdbClient, "lstm", "f_t"); + print_stats(pdbClient, "lstm", "c_t_1"); + print_stats(pdbClient, "lstm", "i_t"); + print_stats(pdbClient, "lstm", "c_t_temp"); + print_stats(pdbClient, "lstm", "c_t"); + } + + { + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + pdb::Handle o_t = + pdb::makeObject("lstm", "o_t"); + pdb::Handle c_t_1 = + pdb::makeObject("lstm", "c_t"); + + // multiply + Handle hidden_state_join = makeObject(); + hidden_state_join->setInput(0, o_t); + hidden_state_join->setInput(1, c_t_1); + + Handle hidden_state_agg = makeObject(); + hidden_state_agg->setInput(hidden_state_join); + + // make the writer + pdb::Handle myWriter = pdb::makeObject("lstm", "h_t"); + myWriter->setInput(hidden_state_agg); + + // run the computation + if (!pdbClient.executeComputations(errMsg, myWriter)) { + std::cout << "Computation failed. Message was: " << errMsg << "\n"; + return 1; + } + } + + { + const pdb::UseTemporaryAllocationBlock tempBlock{1024 * 1024 * 128}; + + print_stats(pdbClient, "lstm", "o_t"); + print_stats(pdbClient, "lstm", "c_t"); + print_stats(pdbClient, "lstm", "h_t"); + } + + + return 0; +}