From 917d9110ef1401c82dc9e23ae5fb67f6b4163c40 Mon Sep 17 00:00:00 2001 From: karanchahal Date: Mon, 8 Apr 2019 13:51:52 +0530 Subject: [PATCH 1/7] SGD Optimiser prelim version added --- main.cpp | 21 ++++++++------ optims/optim.h | 30 ++++++++++++++++++++ optims/sgd.h | 69 ++++++++++++++++++++++++++++++++++++++++++++++ overloads/matrix.h | 8 ++++++ overloads/vector.h | 23 ++++++++++++++++ types/matrix.h | 9 ++++++ types/tensor.h | 35 ++++++++++++----------- utils/matrix.h | 6 ++++ 8 files changed, 176 insertions(+), 25 deletions(-) create mode 100644 optims/optim.h create mode 100644 optims/sgd.h diff --git a/main.cpp b/main.cpp index 54d6f67..9d612b0 100644 --- a/main.cpp +++ b/main.cpp @@ -153,19 +153,22 @@ void updatedSigmoidtest() { } +#include "optims/sgd.h" int main() { - Tensor a({2},{1}); - Tensor b({4},{1}); + Tensor* a = new Tensor({2},{1}); + Tensor* b = new Tensor({4},{1}); + auto loss = tensorOps::add(a,b); + cout<val<backward(); - auto loss = a+b; - cout< sgd; + + // Minimises all tensors wrt to loss + // after updating values with gradients + // it resets all gradients + sgd.minimise(loss,0.1); - oldSigmoidTest(); - newSigmoidTest(); - sigmoidPointerTest(); - updatedSigmoidtest(); } diff --git a/optims/optim.h b/optims/optim.h new file mode 100644 index 0000000..e68a410 --- /dev/null +++ b/optims/optim.h @@ -0,0 +1,30 @@ +/* + This file defines the Base Class for all Optimizers +*/ + +#include "types/tensor.h" +#include "set" + +#ifndef __OPTIM_BASE_INCLUDED__ +#define __OPTIM_BASE_INCLUDED__ + +template +class Optimizer { + + public: + set*> params; + + Optimizer() { + + } + + void zeroGrad() { + for(auto i : params) { + i->zeroGrad(); + } + } + + virtual void step(T learning_rate) {}; +}; + +#endif \ No newline at end of file diff --git a/optims/sgd.h b/optims/sgd.h new file mode 100644 index 0000000..9826426 --- /dev/null +++ b/optims/sgd.h @@ -0,0 +1,69 @@ +/* + This file defines the Stochastic Gradient Descent Optimiser +*/ + +#include "optims/optim.h" +#include "queue" + +#ifndef __OPTIM_SGD_INCLUDED__ +#define __OPTIM_SGD_INCLUDED__ + + +template +class SGD : public Optimizer { + + public: + + SGD() { + this->params.clear(); + } + + void getParams(Tensor* x) { + + queue*> q; + q.push(x); + + while(!q.empty()) { + + auto v = q.front(); + q.pop(); + auto op = v->backOp; + + if(op) { + + if(op->t1 != NULL && this->params.find(op->t1) == this->params.end()) { + q.push(op->t1); + this->params.insert(op->t1); + } + + if(op->t2 != NULL && this->params.find(op->t2) == this->params.end()) { + q.push(op->t2); + this->params.insert(op->t2); + } + } + } + } + + void minimise(Tensor* x, T lr) { + + // Get all tensors in computational grqaph + getParams(x); + + // step through 1 parameter update + step(lr); + + // reset Gradients to zero + this->zeroGrad(); + + } + + // Perform 1 step of learniung rate + void step(T learning_rate) { + for(auto t: this->params) { + t->val = t->val - learning_rate*t->grad; + } + } + +}; + +#endif \ No newline at end of file diff --git a/overloads/matrix.h b/overloads/matrix.h index dfb5dc8..489b124 100644 --- a/overloads/matrix.h +++ b/overloads/matrix.h @@ -46,4 +46,12 @@ Matrix operator + (const T t, const Matrix &rhs) { return Matrix(res, resShape); } +// Subtraction with a scalar +template +Matrix operator - (const T t, const Matrix &rhs) { + auto res = t-rhs.val; + auto resShape = rhs.shape; + return Matrix(res, resShape); +} + #endif \ No newline at end of file diff --git a/overloads/vector.h b/overloads/vector.h index 9aa14ba..8d8e7c0 100644 --- a/overloads/vector.h +++ b/overloads/vector.h @@ -62,6 +62,29 @@ vector operator + (T a, const vector &b) { return arr; } +// Subtraction +template +vector operator - (vector &a, const vector &b) { + assert("Tensors are not of the same size !" && a.size() == b.size()); + vector arr; + for(int i = 0;i +vector operator - (T a, const vector &b) { + vector arr; + for(int i = 0;i vector operator / (vector &a, const vector &b) { diff --git a/types/matrix.h b/types/matrix.h index 6c87200..f1a3ac4 100644 --- a/types/matrix.h +++ b/types/matrix.h @@ -245,6 +245,15 @@ struct Matrix{ return Matrix(res, resShape); } + // Performs elementwise subtraction + Matrix operator - (const Matrix &rhs) { + assert("Shapes aren't compatible for addition !" && + verifyShapeForElementwiseOperation(this->shape, rhs.shape)); + + auto res = this->val - rhs.val; + auto resShape = this->shape; + return Matrix(res, resShape); + } // Performs elementwise division Matrix operator / (const Matrix &rhs) { diff --git a/types/tensor.h b/types/tensor.h index bae4ec1..feb2853 100644 --- a/types/tensor.h +++ b/types/tensor.h @@ -25,26 +25,13 @@ #include "operations/exponentOperation.h" #include "operations/dotOperation.h" #include "operations/sigmoidOperation.h" +#include "utils/matrix.h" #ifndef __TENSOR_FLOAT_INCLUDED__ #define __TENSOR_FLOAT_INCLUDED__ template class Tensor { - private: - - /* - This function is called during the initilaisation of Tensor. It sets the value of it's gradients to zero. This is needed as - during backPropogation the same tensor can be used for different operation, hence to calculate it's partial gradients - each individual operation's gradients have to be summed up. Hence we initialise the tensor's gradients to zero. - - See constructor for it's usage. - */ - void zeroGrad() { - assert(val.shape.size() != 0 && "The value of matrix cannot be uninitialised during initialisng zeros in tensor's gradient"); - vector g(val.val.size(), 0); - this->grad = Matrix(g, val.shape); - } public: @@ -145,6 +132,19 @@ class Tensor { } } + /* + This function is called during the initilaisation of Tensor. It sets the value of it's gradients to zero. This is needed as + during backPropogation the same tensor can be used for different operation, hence to calculate it's partial gradients + each individual operation's gradients have to be summed up. Hence we initialise the tensor's gradients to zero. + + See constructor for it's usage. + */ + void zeroGrad() { + assert(val.shape.size() != 0 && "The value of matrix cannot be uninitialised during initialisng zeros in tensor's gradient"); + vector g(val.val.size(), 0); + this->grad = Matrix(g, val.shape); + } + /* From here on, we overload the operators like +, / and * to define what happens when we we add, divide and multiply tensors. We also support other operations like dot @@ -221,9 +221,12 @@ class Tensor { } // Destructor + // Deletes all dependencies to this tensor + // TODO ~Tensor() { - // delete backOp; - // delete frontOp; + // Go back towards computational graph + // delete every Tensor and Op encountered in a DFS fashion + } }; diff --git a/utils/matrix.h b/utils/matrix.h index fc79d3e..1e8394e 100644 --- a/utils/matrix.h +++ b/utils/matrix.h @@ -24,6 +24,12 @@ namespace utils { return Matrix(m.val,shape); } + + template< typename T> + Matrix zerosLike(const Matrix &m) { + vector val(m.val.size(),0); + return Matrix(val,m.shape); + } } #endif \ No newline at end of file From 855efbf9c510e993cdc49b9c4a83acb210d7956e Mon Sep 17 00:00:00 2001 From: karanchahal Date: Mon, 8 Apr 2019 17:23:20 +0530 Subject: [PATCH 2/7] Added tests for SGD and wrote some memory safety/memory leak saving code --- main.cpp | 8 ++++--- operations/operation.h | 5 ++++ optims/optim.h | 5 ++-- optims/sgd.h | 12 ++++++---- tests/dense.h | 7 +++++- tests/main.cpp | 1 + tests/sgd.h | 54 ++++++++++++++++++++++++++++++++++++++++++ tests/tensor.h | 26 ++++++++++++++++---- types/tensor.h | 2 +- 9 files changed, 105 insertions(+), 15 deletions(-) create mode 100644 tests/sgd.h diff --git a/main.cpp b/main.cpp index 9d612b0..b65f6cb 100644 --- a/main.cpp +++ b/main.cpp @@ -151,6 +151,7 @@ void updatedSigmoidtest() { cout<grad<val<backward(); - SGD sgd; + SGD sgd(0.1); // Minimises all tensors wrt to loss // after updating values with gradients // it resets all gradients - sgd.minimise(loss,0.1); - + sgd.minimise(loss); + // Will recursively delete all tensors & ops attached to it + delete loss; } diff --git a/operations/operation.h b/operations/operation.h index 0a3f721..901acb1 100644 --- a/operations/operation.h +++ b/operations/operation.h @@ -40,6 +40,11 @@ class Operation { // New API for forward Prop virtual Tensor* forward() = 0; + + ~Operation() { + delete t1; + delete t2; + } }; diff --git a/optims/optim.h b/optims/optim.h index e68a410..91c26d6 100644 --- a/optims/optim.h +++ b/optims/optim.h @@ -3,7 +3,7 @@ */ #include "types/tensor.h" -#include "set" +#include "unordered_set" #ifndef __OPTIM_BASE_INCLUDED__ #define __OPTIM_BASE_INCLUDED__ @@ -12,7 +12,8 @@ template class Optimizer { public: - set*> params; + unordered_set*> params; + T lr; Optimizer() { diff --git a/optims/sgd.h b/optims/sgd.h index 9826426..0381aeb 100644 --- a/optims/sgd.h +++ b/optims/sgd.h @@ -14,8 +14,9 @@ class SGD : public Optimizer { public: - SGD() { + SGD(T lr) { this->params.clear(); + this->lr = lr; } void getParams(Tensor* x) { @@ -44,24 +45,27 @@ class SGD : public Optimizer { } } - void minimise(Tensor* x, T lr) { + void minimise(Tensor* x) { // Get all tensors in computational grqaph getParams(x); // step through 1 parameter update - step(lr); + step(this->lr); // reset Gradients to zero this->zeroGrad(); } - // Perform 1 step of learniung rate + // Perform 1 step of learning rate void step(T learning_rate) { + for(auto t: this->params) { t->val = t->val - learning_rate*t->grad; } + + this->params.clear(); // Clear out old params. Should we do this ? } }; diff --git a/tests/dense.h b/tests/dense.h index 18b9f8b..4072a94 100644 --- a/tests/dense.h +++ b/tests/dense.h @@ -15,11 +15,14 @@ TEST(DENSE_LAYER_TESTS, SHAPE_CHECKS) { Tensor* x1 = new Tensor({1,2},{1,2}); // put 1 by 2 tensor auto m = fc1.forward(x1); // should work fine - Tensor* x2 = new Tensor({1},{1}); // put 1 by 2 tensor + delete m; ASSERT_DEATH({ + Tensor* x2 = new Tensor({1},{1}); // put 1 by 2 tensor + Dense fc1(2,5); // input - 2, output should be 5 auto m = fc1.forward(x2); // should give error as dot product will not be compatible ! }, "Shapes aren't compatible for dot product !"); + } /* @@ -41,4 +44,6 @@ TEST(DENSE_LAYER_TESTS, CORRECTNESS_CHECK) { auto expectedVal = matrixOps::sigmoid((x->val).dot(w) + b); ASSERT_TRUE(testUtils::isMatrixEqual(m->val, expectedVal)); + + delete m; } diff --git a/tests/main.cpp b/tests/main.cpp index cde9948..6065f3c 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -7,6 +7,7 @@ #include "tests/matrix.h" #include "tests/tensor.h" #include "tests/dense.h" +#include "tests/sgd.h" int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); diff --git a/tests/sgd.h b/tests/sgd.h new file mode 100644 index 0000000..3c40d02 --- /dev/null +++ b/tests/sgd.h @@ -0,0 +1,54 @@ +/* + This file tests the SGD Optimizer layer. +*/ + +#include +#include "optims/sgd.h" +#include "tests/utils.h" +#include "overloads/tensor.h" + +/* + Tests that the optimizer layer gets all the tensors that need to be updated. +*/ +TEST(SGD_OPTIM_TESTS, TENSOR_UPDATE_CHECK) { + Tensor* a = new Tensor({2},{1}); + Tensor* b = new Tensor({4},{1}); + auto c = tensorOps::add(a,b); + Tensor* d = new Tensor({3},{1}); + + auto e = tensorOps::multiply(c,d); + e->backward(); + + SGD sgd(0.1); + // get all paramters/tensors that need to be updated wrt to e + sgd.getParams(e); + unordered_set*> expected_res = {a,b,c,d}; + ASSERT_TRUE(sgd.params == expected_res); + + // Clean up + delete e; +} + +/* + Tests that the tensor values are updated according to gradient values and learning rate +*/ +TEST(SGD_OPTIM_TESTS, SGD_STEP_CHECK) { + Tensor* a = new Tensor({2},{1}); + Tensor* b = new Tensor({4},{1}); + auto c = tensorOps::add(a,b); + Tensor* d = new Tensor({3},{1}); + + auto e = tensorOps::multiply(c,d); + e->backward(); + + SGD sgd(1); + // get all paramters/tensors that need to be updated wrt to e + sgd.minimise(e); + + ASSERT_TRUE(a->val.val[0] == -1); // update = 2 - 1*3 + ASSERT_TRUE(b->val.val[0] == 1); // update = 4 - 1*3 + ASSERT_TRUE(d->val.val[0] == -3); // update = 3 -1*6 + + // Clean up + delete e; +} diff --git a/tests/tensor.h b/tests/tensor.h index e97ef9a..3c2e477 100644 --- a/tests/tensor.h +++ b/tests/tensor.h @@ -16,7 +16,7 @@ TEST( TENSOR_TESTS, TensorCreation) { ASSERT_DEATH({ vector a({1,2,3,4,5,6}); vector shape1({2,4}); - Matrix m1(a,shape1); + Tensor m1(a,shape1); }, "Shape and size of vector are incompatible !"); // testing for no asserts with various dimensions that can used in nd matrix @@ -24,9 +24,9 @@ TEST( TENSOR_TESTS, TensorCreation) { vector shape1({2,3}); vector shape2({1,1,1,2,3}); vector shape3({2,3,1,1,1}); - Matrix m1(a,shape1); - m1 = Matrix(a,shape2); - m1 = Matrix(a,shape3); + Tensor m1(a,shape1); + m1 = Tensor(a,shape2); + m1 = Tensor(a,shape3); } /* @@ -42,6 +42,9 @@ TEST( TENSOR_TESTS, TensorAddOperations) { Matrix res({2,4,6,8,10},{5}); ASSERT_TRUE(testUtils::isMatrixEqual(ans->val,res)); + + // Clean up + delete ans; } @@ -53,6 +56,9 @@ TEST( TENSOR_TESTS, TensorMultiplyOperations) { Matrix res({1,4,9,16,25},{5}); ASSERT_TRUE(testUtils::isMatrixEqual(ans->val,res)); + + // Clean up + delete ans; } TEST( TENSOR_TESTS, TensorDivideOperations) { @@ -63,6 +69,9 @@ TEST( TENSOR_TESTS, TensorDivideOperations) { Matrix res({5,2,5,2,1},{5}); ASSERT_TRUE(testUtils::isMatrixEqual(ans->val,res)); + + // Clean up + delete ans; } /* @@ -90,6 +99,9 @@ TEST( TENSOR_TESTS, TensorSigmoidOperations) { Matrix resGrad({0.196611926}, {1}); ASSERT_TRUE(testUtils::isMatrixEqual(one->grad,resGrad)); // check back Propogation + + // Clean up + delete ans; } /* @@ -211,6 +223,9 @@ TEST( TENSOR_TESTS, ComputationGraph) { ASSERT_TRUE(x1->frontOp == b->backOp); ASSERT_TRUE(x1->backOp == NULL); + // Clean up + delete k; + } /* @@ -258,4 +273,7 @@ TEST(TENSOR_TESTS, BackwardPropogation) { res = Matrix({0.196611971},{1}); ASSERT_TRUE(testUtils::isMatrixEqual(w3->grad,res)); + + // Clean up + delete k; } diff --git a/types/tensor.h b/types/tensor.h index feb2853..e10698d 100644 --- a/types/tensor.h +++ b/types/tensor.h @@ -226,7 +226,7 @@ class Tensor { ~Tensor() { // Go back towards computational graph // delete every Tensor and Op encountered in a DFS fashion - + delete backOp; } }; From 0979967b1d07fa7bf8a7e8e53c6d5af063ee7558 Mon Sep 17 00:00:00 2001 From: karanchahal Date: Tue, 9 Apr 2019 12:04:30 +0530 Subject: [PATCH 3/7] Added comments for SGD --- optims/sgd.h | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/optims/sgd.h b/optims/sgd.h index 0381aeb..58e1de7 100644 --- a/optims/sgd.h +++ b/optims/sgd.h @@ -1,5 +1,18 @@ /* - This file defines the Stochastic Gradient Descent Optimiser + This file defines the Stochastic Gradient Descent Optimiser. The Stochastic Gradient Descent + Optimizer takes the loss computed over a single training example or the averages of the loss + computed with multiple training examples and "minimises" the loss. + + By minimising, we mean it finds out all the updatable tensors that contributed towards + computing this loss. Once it has these parameters it performs an update step on each + parameter (Tensor) to tweak them into the right direction to minimise the overall loss. + + It performs this update step by this formula: + + val = val - learning_rate*gradient_of_val + + Where val is the value of the tensor and gradient_of_val is the partial gradient of the + tensor with respect to the loss. */ #include "optims/optim.h" @@ -19,6 +32,15 @@ class SGD : public Optimizer { this->lr = lr; } + /* + This function does a full search through the computational graph of the Tensor x and + stores all the Tensor nodes of the graph in the params set. + + The params set represents all the tensors that need t be updated. + + As of now, a BFS style algorithm traverses through the graph to find out all the Tensor + nodes. + */ void getParams(Tensor* x) { queue*> q; @@ -45,9 +67,16 @@ class SGD : public Optimizer { } } + /* + This function is the function all users will use to perfrom the gradient descent update + for their model. It perfroms this operation in 3 phases. + 1. gets all tensor parameters + 2. Updates all these paramaters via the step function + 3. Clear's all the gradients of the parameters for the next step. + */ void minimise(Tensor* x) { - // Get all tensors in computational grqaph + // Get all tensors in computational graph getParams(x); // step through 1 parameter update @@ -58,7 +87,7 @@ class SGD : public Optimizer { } - // Perform 1 step of learning rate + // Performs 1 step of gradient descent. See top of the file to see definition of SGD. void step(T learning_rate) { for(auto t: this->params) { From b8a9b1989fa17180b256109209df49afc28c3b31 Mon Sep 17 00:00:00 2001 From: karanchahal Date: Tue, 9 Apr 2019 12:05:40 +0530 Subject: [PATCH 4/7] Added celsius 2 faranheit dataset --- data/celsius2faranheit.h | 31 +++++++++++++++++++++++++++++++ data/dataloader.h | 16 ++++++++++++++++ main.cpp | 24 +++++++++--------------- 3 files changed, 56 insertions(+), 15 deletions(-) create mode 100644 data/celsius2faranheit.h create mode 100644 data/dataloader.h diff --git a/data/celsius2faranheit.h b/data/celsius2faranheit.h new file mode 100644 index 0000000..b50d479 --- /dev/null +++ b/data/celsius2faranheit.h @@ -0,0 +1,31 @@ +#include "data/dataloader.h" +#include + +#ifndef __C2F_DATASET_INCLUDED__ +#define __C2F_DATASET_INCLUDED__ + +template +class Celsius2Faranheit: public DataLoader { + + private: + int MAX_CELSIUS = 10; + + T toFaranheit(I input) { + return (9*input)/5 + 32; + } + + public: + void add(I input, T target) { + this->dataset.push_back(make_pair(input,target)); + } + + void create(int num_examples) { + for(int i=0; i< num_examples;i++) { + I input = rand() % MAX_CELSIUS + 1; // random int value between 1 and MAX_CELSIUS + T target = toFaranheit(input); + add(input,target); + } + } +}; + +#endif diff --git a/data/dataloader.h b/data/dataloader.h new file mode 100644 index 0000000..4889ff9 --- /dev/null +++ b/data/dataloader.h @@ -0,0 +1,16 @@ +#include +#include + +#ifndef __DATALOADER_INCLUDED__ +#define __DATALOADER_INCLUDED__ + +template +class DataLoader { + + public: + vector> dataset; + + virtual void add(I input, T target) = 0; +}; + +#endif diff --git a/main.cpp b/main.cpp index b65f6cb..ad68797 100644 --- a/main.cpp +++ b/main.cpp @@ -155,22 +155,16 @@ void updatedSigmoidtest() { } #include "optims/sgd.h" +#include "data/celsius2faranheit.h" int main() { - Tensor* a = new Tensor({2},{1}); - Tensor* b = new Tensor({4},{1}); - auto loss = tensorOps::add(a,b); - cout<val<backward(); - - SGD sgd(0.1); - - // Minimises all tensors wrt to loss - // after updating values with gradients - // it resets all gradients - sgd.minimise(loss); - - // Will recursively delete all tensors & ops attached to it - delete loss; + Celsius2Faranheit dataset; + dataset.create(5); + for(auto i: dataset.dataset) { + auto inp = i.first; + auto tar = i.second; + + cout<<"Input: "< Date: Wed, 10 Apr 2019 21:52:23 +0530 Subject: [PATCH 5/7] Big Commit: Added the data loader, a celcius to faranheit dataloader, the power operation support for tensors and matrices, working SGD support and a full working example of a neural network that trains successfuly to learn how to convert celcius to faranheit --- buildTensorflow.h | 2 + data/celsius2faranheit.h | 2 +- data/dataloader.h | 4 +- main.cpp | 200 +++++++------------------------ operations/operations_Impl.h | 4 +- operations/powerOperation.h | 28 +++++ operations/powerOperation_Impl.h | 44 +++++++ optims/sgd.h | 4 +- overloads/matrix.h | 10 +- overloads/tensor.h | 7 ++ tests/matrix.h | 33 +++-- tests/sigmoidTests.h | 155 ++++++++++++++++++++++++ tests/tensor.h | 22 ++++ types/tensor.h | 2 +- 14 files changed, 339 insertions(+), 178 deletions(-) create mode 100644 operations/powerOperation.h create mode 100644 operations/powerOperation_Impl.h create mode 100644 tests/sigmoidTests.h diff --git a/buildTensorflow.h b/buildTensorflow.h index cf2cf4c..a15b992 100644 --- a/buildTensorflow.h +++ b/buildTensorflow.h @@ -2,3 +2,5 @@ #include "overloads/tensor.h" #include "operations/operations_Impl.h" #include "layers/dense.h" +#include "optims/sgd.h" +#include "data/celsius2faranheit.h" \ No newline at end of file diff --git a/data/celsius2faranheit.h b/data/celsius2faranheit.h index b50d479..37fe2cb 100644 --- a/data/celsius2faranheit.h +++ b/data/celsius2faranheit.h @@ -16,7 +16,7 @@ class Celsius2Faranheit: public DataLoader { public: void add(I input, T target) { - this->dataset.push_back(make_pair(input,target)); + this->data.push_back(make_pair(input,target)); } void create(int num_examples) { diff --git a/data/dataloader.h b/data/dataloader.h index 4889ff9..cb1266e 100644 --- a/data/dataloader.h +++ b/data/dataloader.h @@ -8,9 +8,9 @@ template class DataLoader { public: - vector> dataset; + vector> data; virtual void add(I input, T target) = 0; -}; +}; #endif diff --git a/main.cpp b/main.cpp index ad68797..283c85b 100644 --- a/main.cpp +++ b/main.cpp @@ -1,170 +1,52 @@ #include "buildTensorflow.h" -void oldSigmoidTest() { - - Tensor w0({2},{1}); - Tensor x0({-1},{1}); - - Tensor w1({-3},{1}); - Tensor x1({-2},{1}); - - Tensor w3({-3},{1}); - - Tensor a = w0*x0; - Tensor b = w1*x1; - Tensor c = a + b; - Tensor d = w3+c; - Tensor e({-1}, {1}); - Tensor f = d*e; - Tensor g = f.exp(); - Tensor h({1}, {1}); - Tensor i = g + h; - Tensor j({1}, {1}); - Tensor k = j/i; - - vector vsl = {1}; - vector sh = {1}; - auto grad = Matrix(vsl,sh); - k.backward(grad); - - - cout< w0({2},{1}); - Tensor x0({-1},{1}); - - Tensor w1({-3},{1}); - Tensor x1({-2},{1}); - - Tensor w3({-3},{1}); - Tensor e({-1}, {1}); - Tensor h({1}, {1}); - Tensor j({1}, {1}); - - Tensor a = e*(w0*x0 + w1*x1 + w3); - Tensor k = j/(a.exp() + h); - - vector vsl = {1}; - vector sh = {1}; - auto grad = Matrix(vsl,sh); - k.backward(grad); - - cout<* w0 = new Tensor({2},{1}); - Tensor* x0= new Tensor({-1},{1}); - - Tensor* w1= new Tensor({-3},{1}); - Tensor* x1= new Tensor({-2},{1}); - - Tensor* w3= new Tensor({-3},{1}); - - auto a = tensorOps::multiply(w0,x0); - auto b = tensorOps::multiply(w1,x1); - auto c = tensorOps::add(a,b); - auto d = tensorOps::add(w3,c); - - Tensor* e = new Tensor({-1}, {1}); - auto f = tensorOps::multiply(d,e); - - auto g = tensorOps::exp(f); // exponent - - Tensor* h = new Tensor({1}, {1}); - auto i = tensorOps::add(g,h); - - Tensor* j = new Tensor({1}, {1}); - auto k = tensorOps::divide(j,i); - - auto grad = Matrix({1},{1}); - k->backward(grad); - - - cout<grad<grad<grad<grad<grad<* w0 = new Tensor({2},{1}); - Tensor* x0= new Tensor({-1},{1}); +// Example of training a network on the buildTensorflow framework. +int main() { + // Load Dataset + Celsius2Faranheit dataset; + dataset.create(5); - Tensor* w1= new Tensor({-3},{1}); - Tensor* x1= new Tensor({-2},{1}); + // Create Model + Dense fc1(1,1,NO_ACTIVATION); - Tensor* w3= new Tensor({-3},{1}); + // Initialise Optimiser + SGD sgd(0.01); - auto a = tensorOps::multiply(w0,x0); - auto b = tensorOps::multiply(w1,x1); - auto c = tensorOps::add(a,b); - auto d = tensorOps::add(w3,c); - - auto k = tensorOps::sigmoid(d); - k->backward(); - - cout<grad<grad<grad<grad<grad<({i.first}, {1,1}); + auto tar = new Tensor({i.second}, {1,1}); + + // Forward Prop + auto out = fc1.forward(inp); + + // Get Loss + auto l = new Tensor({-1}, {1,1}); + auto k = tensorOps::multiply(l,tar); + auto loss = tensorOps::add(out,k); // error in loss + auto finalLoss = tensorOps::power(loss,(float)2); + + // Compute backProp + finalLoss->backward(); + // cout<val<({cel}, {1,1}); + auto out1 = fc1.forward(test); -int main() { - Celsius2Faranheit dataset; - dataset.create(5); - for(auto i: dataset.dataset) { - auto inp = i.first; - auto tar = i.second; + cout<<"The conversion of "<val< +class PowerOperation : public Operation { + public: + T pow; + + PowerOperation(Tensor *t1, T pow) { + this->t1 = t1; + this->pow = pow; + } + void backward(Matrix grad); + + Tensor forwardDeprecated(); + + Tensor* forward(); +}; + +#endif + diff --git a/operations/powerOperation_Impl.h b/operations/powerOperation_Impl.h new file mode 100644 index 0000000..f16f05e --- /dev/null +++ b/operations/powerOperation_Impl.h @@ -0,0 +1,44 @@ +/* + This file contains the implementation of the forward and backward pass of + the power operation. +*/ + +#include "operations/powerOperation.h" + +#ifndef __OP_IMPL_POWER_INCLUDED__ +#define __OP_IMPL_POWER_INCLUDED__ + +/* + Backpropogation of the power operation. + + F = x*pow is forward propogation + The gradient would be as follows: + 1. dF/dx = pow*x^(pow-1) +*/ +template +void PowerOperation::backward(Matrix grad) { + this->t1->backward(grad * (pow * matrixOps::power(this->t1->val,pow-1))); +} + +/* + Forward Propogation of the operation. Returns a tensor. + + TODO: Remove: See addition operation impl for more details +*/ +template +Tensor PowerOperation::forwardDeprecated() { + return NULL; +} + +/* + Forward Propogation of the operation. Return pointer to the tensor. + Forward propogation is simply y = x^(pow). +*/ +template +Tensor* PowerOperation::forward() { + this->t3 = new Tensor(matrixOps::power(this->t1->val, this->pow), this); + return this->t3; +} + +#endif + diff --git a/optims/sgd.h b/optims/sgd.h index 58e1de7..ddbc498 100644 --- a/optims/sgd.h +++ b/optims/sgd.h @@ -43,6 +43,8 @@ class SGD : public Optimizer { */ void getParams(Tensor* x) { + this->params.clear(); // Clear out old params. Should we do this ? + queue*> q; q.push(x); @@ -93,8 +95,6 @@ class SGD : public Optimizer { for(auto t: this->params) { t->val = t->val - learning_rate*t->grad; } - - this->params.clear(); // Clear out old params. Should we do this ? } }; diff --git a/overloads/matrix.h b/overloads/matrix.h index 489b124..2ebe183 100644 --- a/overloads/matrix.h +++ b/overloads/matrix.h @@ -7,12 +7,20 @@ #ifndef __MATRIX_OPS_INCLUDED__ #define __MATRIX_OPS_INCLUDED__ -// Sigmoid + namespace matrixOps { + + // Sigmoid Operation template Matrix sigmoid(const Matrix &a) { return (T)1/((T)1 + (((T)-1)*a).exp()); } + + // Power Operation + template + Matrix power(Matrix &a, T pow) { + return a^pow; + } }; // Overloaded function for printing matrix: cout<frontOp->forward(); } + // Power + template + Tensor* power(Tensor* one, T t) { + one->frontOp = new PowerOperation(one, t); + return one->frontOp->forward(); + } + }; #endif diff --git a/tests/matrix.h b/tests/matrix.h index 8aa14bd..1e4dc54 100644 --- a/tests/matrix.h +++ b/tests/matrix.h @@ -127,6 +127,28 @@ TEST( MATRIX_TESTS, MatrixOperationMultiplicationCheck) { ASSERT_TRUE(testUtils::isMatrixEqual(ans,res)); } +/* + This test tests the accuracy of the power operation between a matrix and a scalar +*/ +TEST( MATRIX_TESTS, MatrixOperationPowerCheck) { + + vector a({1,2,3}); + vector shape1({1,3}); + Matrix m1(a,shape1); + int pow = 3; + auto ans = m1^pow; // Checking barebones operation + Matrix res({1,8,27},{1,3}); + + ASSERT_TRUE(testUtils::isMatrixEqual(ans,res)); + + Matrix m2({1,2,3},{1,3}); + pow = 2; + Matrix res2({1,4,9},{1,3}); + auto ans2 = matrixOps::power(m2,pow); // Checking wrapper function + + ASSERT_TRUE(testUtils::isMatrixEqual(ans2,res2)); +} + /* This test tests the accuracy of the division operation between 2 matrices */ @@ -143,17 +165,6 @@ TEST( MATRIX_TESTS, MatrixOperationDivisionCheck) { ASSERT_TRUE(testUtils::isMatrixEqual(ans,res)); } -/* - This test tests the accuracy of the power operation between matrice and scalar -*/ -TEST( MATRIX_TESTS, MatrixOperationPowerCheck) { - vector a({1,2,3}); - vector shape1({1,3}); - Matrix m1(a,shape1); - auto ans = m1^2; - Matrix res({1,4,9},{1,3}); - ASSERT_TRUE(testUtils::isMatrixEqual(ans,res)); -} /* This test tests the accuracy of the exponent operation. diff --git a/tests/sigmoidTests.h b/tests/sigmoidTests.h new file mode 100644 index 0000000..8d9a36a --- /dev/null +++ b/tests/sigmoidTests.h @@ -0,0 +1,155 @@ +#include "buildTensorflow.h" + +void oldSigmoidTest() { + + Tensor w0({2},{1}); + Tensor x0({-1},{1}); + + Tensor w1({-3},{1}); + Tensor x1({-2},{1}); + + Tensor w3({-3},{1}); + + Tensor a = w0*x0; + Tensor b = w1*x1; + Tensor c = a + b; + Tensor d = w3+c; + Tensor e({-1}, {1}); + Tensor f = d*e; + Tensor g = f.exp(); + Tensor h({1}, {1}); + Tensor i = g + h; + Tensor j({1}, {1}); + Tensor k = j/i; + + vector vsl = {1}; + vector sh = {1}; + auto grad = Matrix(vsl,sh); + k.backward(grad); + + + cout< w0({2},{1}); + Tensor x0({-1},{1}); + + Tensor w1({-3},{1}); + Tensor x1({-2},{1}); + + Tensor w3({-3},{1}); + Tensor e({-1}, {1}); + Tensor h({1}, {1}); + Tensor j({1}, {1}); + + Tensor a = e*(w0*x0 + w1*x1 + w3); + Tensor k = j/(a.exp() + h); + + vector vsl = {1}; + vector sh = {1}; + auto grad = Matrix(vsl,sh); + k.backward(grad); + + cout<* w0 = new Tensor({2},{1}); + Tensor* x0= new Tensor({-1},{1}); + + Tensor* w1= new Tensor({-3},{1}); + Tensor* x1= new Tensor({-2},{1}); + + Tensor* w3= new Tensor({-3},{1}); + + auto a = tensorOps::multiply(w0,x0); + auto b = tensorOps::multiply(w1,x1); + auto c = tensorOps::add(a,b); + auto d = tensorOps::add(w3,c); + + Tensor* e = new Tensor({-1}, {1}); + auto f = tensorOps::multiply(d,e); + + auto g = tensorOps::exp(f); // exponent + + Tensor* h = new Tensor({1}, {1}); + auto i = tensorOps::add(g,h); + + Tensor* j = new Tensor({1}, {1}); + auto k = tensorOps::divide(j,i); + + auto grad = Matrix({1},{1}); + k->backward(grad); + + + cout<grad<grad<grad<grad<grad<* w0 = new Tensor({2},{1}); + Tensor* x0= new Tensor({-1},{1}); + + Tensor* w1= new Tensor({-3},{1}); + Tensor* x1= new Tensor({-2},{1}); + + Tensor* w3= new Tensor({-3},{1}); + + auto a = tensorOps::multiply(w0,x0); + auto b = tensorOps::multiply(w1,x1); + auto c = tensorOps::add(a,b); + auto d = tensorOps::add(w3,c); + + auto k = tensorOps::sigmoid(d); + k->backward(); + + cout<grad<grad<grad<grad<grad<* one = new Tensor({2,3,4},{1,3}); + float pow = 3; + auto ans = tensorOps::power(one,pow); + Matrix res({8,27,64}, {1,3}); + + ASSERT_TRUE(testUtils::isMatrixEqual(ans->val,res)); // check front Propogation + + ans->backward(); + + Matrix resGrad({12,27,48}, {1,3}); + ASSERT_TRUE(testUtils::isMatrixEqual(one->grad,resGrad)); // check back Propogation + + // Clean up + delete ans; +} + + /* Test Computational Graph by checking Pointer Values of each tensor and operation for a barebones sigmoid function diff --git a/types/tensor.h b/types/tensor.h index e10698d..1e61f16 100644 --- a/types/tensor.h +++ b/types/tensor.h @@ -25,7 +25,7 @@ #include "operations/exponentOperation.h" #include "operations/dotOperation.h" #include "operations/sigmoidOperation.h" -#include "utils/matrix.h" +#include "operations/powerOperation.h" #ifndef __TENSOR_FLOAT_INCLUDED__ #define __TENSOR_FLOAT_INCLUDED__ From fa3f7514483af6f6f5970187bed5a0f6f7aed8c8 Mon Sep 17 00:00:00 2001 From: karanchahal Date: Wed, 10 Apr 2019 22:14:26 +0530 Subject: [PATCH 6/7] Added comments to code --- data/celsius2faranheit.h | 25 +++++++++++++++++++++++++ data/dataloader.h | 7 +++++++ main.cpp | 2 ++ optims/optim.h | 8 +++++++- optims/sgd.h | 6 +++--- types/tensor.h | 11 ++++++----- 6 files changed, 50 insertions(+), 9 deletions(-) diff --git a/data/celsius2faranheit.h b/data/celsius2faranheit.h index 37fe2cb..1964abb 100644 --- a/data/celsius2faranheit.h +++ b/data/celsius2faranheit.h @@ -1,3 +1,24 @@ +/* + This file defines the Celsius To Faranheit DataLoader. It's input is variables containing the + celsius numbers and the targets are the corresponding faranheit numbers. + + The way to use this dataset is as follows: + + Celsius2Faranheit dataloader; + dataloader.create(10); // Creates 10 training examples + + for(auto i: dataloader.data) { + auto inp = j.first; + auto tar = j.second; + + // And then use this above data in your model for training or inference + } + + Note that the data won't be outputted in tensors. It will simply be of the data type the user + signifies in the dataloader defination. In the above case the input and targets are both floats. + +*/ + #include "data/dataloader.h" #include @@ -10,15 +31,19 @@ class Celsius2Faranheit: public DataLoader { private: int MAX_CELSIUS = 10; + // Helper function to convert celsius to faranheit T toFaranheit(I input) { return (9*input)/5 + 32; } public: + + // Adds a training example into the dataset void add(I input, T target) { this->data.push_back(make_pair(input,target)); } + // Populates the dataset with the number of examples specified by the user. void create(int num_examples) { for(int i=0; i< num_examples;i++) { I input = rand() % MAX_CELSIUS + 1; // random int value between 1 and MAX_CELSIUS diff --git a/data/dataloader.h b/data/dataloader.h index cb1266e..22d9a8e 100644 --- a/data/dataloader.h +++ b/data/dataloader.h @@ -1,3 +1,8 @@ +/* + This file defines the base class of each Dataset in the project. The data is stored in a simple + vector and each object in the vector is a pari signifying input and target (ground truth). +*/ + #include #include @@ -8,8 +13,10 @@ template class DataLoader { public: + // This variable contains all the data of the dataset vector> data; + // This function perfroms the operation that populates the "data" variable. virtual void add(I input, T target) = 0; }; diff --git a/main.cpp b/main.cpp index 283c85b..b67c9e9 100644 --- a/main.cpp +++ b/main.cpp @@ -48,5 +48,7 @@ int main() { cout<<"The conversion of "<val< class Optimizer { public: + + // This variable contains all the tensors that need to be updated via the optimiser unordered_set*> params; + + // The learning rate T lr; Optimizer() { } + // This function resets the gradients of the tensors in params to zero for the next forward pass void zeroGrad() { for(auto i : params) { i->zeroGrad(); } } + // This overloaded function specifes how one optimisation step will be performed virtual void step(T learning_rate) {}; }; diff --git a/optims/sgd.h b/optims/sgd.h index ddbc498..7c48c9b 100644 --- a/optims/sgd.h +++ b/optims/sgd.h @@ -71,9 +71,9 @@ class SGD : public Optimizer { /* This function is the function all users will use to perfrom the gradient descent update - for their model. It perfroms this operation in 3 phases. - 1. gets all tensor parameters - 2. Updates all these paramaters via the step function + for their model. It performs this operation in 3 phases. + 1. Gets all tensor parameters + 2. Updates all these parameters via the step function 3. Clear's all the gradients of the parameters for the next step. */ void minimise(Tensor* x) { diff --git a/types/tensor.h b/types/tensor.h index 1e61f16..9c41341 100644 --- a/types/tensor.h +++ b/types/tensor.h @@ -220,12 +220,13 @@ class Tensor { return this->frontOp->forwardDeprecated(); } - // Destructor - // Deletes all dependencies to this tensor - // TODO + /* + Go back towards computational graph and deletes every Tensor and Op encountered + in a DFS fashion + + TODO: find better way to clear memory of all tensors and prevent memory leaks. + */ ~Tensor() { - // Go back towards computational graph - // delete every Tensor and Op encountered in a DFS fashion delete backOp; } From d9074063729c36a06364e6d91166b91eb5ec7f2a Mon Sep 17 00:00:00 2001 From: karanchahal Date: Thu, 11 Apr 2019 01:21:18 +0530 Subject: [PATCH 7/7] GPU code added for dot product and created 2 different channels for building cpu and gpu versions --- buildTensorflow.h | 3 ++ buildTensorflowGpu.h | 11 ++++++++ gpu/defn.h | 1 + gpu/{dot.h => dot/defn.h} | 7 +++-- gpu/{dot.cu => dot/impl.cuh} | 29 +++++++++++++------ gpu/impl.h | 1 + layers/dense.h | 3 ++ main.cu | 54 ++++++++++++++++++++++++++++++++++++ types/matrix.h | 3 +- 9 files changed, 98 insertions(+), 14 deletions(-) create mode 100644 buildTensorflowGpu.h create mode 100644 gpu/defn.h rename gpu/{dot.h => dot/defn.h} (62%) rename gpu/{dot.cu => dot/impl.cuh} (63%) create mode 100644 gpu/impl.h create mode 100644 main.cu diff --git a/buildTensorflow.h b/buildTensorflow.h index a15b992..88d31a2 100644 --- a/buildTensorflow.h +++ b/buildTensorflow.h @@ -1,3 +1,6 @@ +// Check whether GPU is accessible or not +bool gpu = false; + #include "types/tensor.h" #include "overloads/tensor.h" #include "operations/operations_Impl.h" diff --git a/buildTensorflowGpu.h b/buildTensorflowGpu.h new file mode 100644 index 0000000..b8d69d1 --- /dev/null +++ b/buildTensorflowGpu.h @@ -0,0 +1,11 @@ +// Check whether GPU is accessible or not +bool gpu = true; + +#include "gpu/defn.h" // Includes GPU Kernel Code Defination for Forward pass +#include "types/tensor.h" +#include "gpu/impl.h" // Includes GPU Kernel Code Implementation +#include "overloads/tensor.h" +#include "operations/operations_Impl.h" +#include "layers/dense.h" +#include "optims/sgd.h" +#include "data/celsius2faranheit.h" \ No newline at end of file diff --git a/gpu/defn.h b/gpu/defn.h new file mode 100644 index 0000000..2d2d39f --- /dev/null +++ b/gpu/defn.h @@ -0,0 +1 @@ +#include "gpu/dot/defn.h" \ No newline at end of file diff --git a/gpu/dot.h b/gpu/dot/defn.h similarity index 62% rename from gpu/dot.h rename to gpu/dot/defn.h index a16db39..9da72d8 100644 --- a/gpu/dot.h +++ b/gpu/dot/defn.h @@ -1,5 +1,7 @@ -#ifndef __GPU_DOT_INCLUDED__ -#define __GPU_DOT_INCLUDED__ +#include "utils/common.h" + +#ifndef __GPU_DOT_DEFN_INCLUDED__ +#define __GPU_DOT_DEFN_INCLUDED__ template struct Matrix; @@ -8,4 +10,3 @@ template void dotGPU(vector &res, const Matrix* lhs, const Matrix &rhs, int start, int startRes); #endif - diff --git a/gpu/dot.cu b/gpu/dot/impl.cuh similarity index 63% rename from gpu/dot.cu rename to gpu/dot/impl.cuh index 6b9bfc8..2f7a7f7 100644 --- a/gpu/dot.cu +++ b/gpu/dot/impl.cuh @@ -1,17 +1,20 @@ -#include "utils/common.h" -#include "types/matrix.h" +#ifndef __GPU_DOT_IMPL_INCLUDED__ +#define __GPU_DOT_IMPL_INCLUDED__ + +// TODO need to refactor this to different files and +// figure out a way to link it for the GPU build template -__global__ void mm(T* a, T* b, T* c, T width) { +__global__ void mm(T* a, T* b, T* c, T width, T second) { int x = blockIdx.x; // block id int y = threadIdx.x; // thread id T temp = 0; for(int i = 0;i< width;i++) { - temp += a[x*width + i]*b[i*width+ y]; + temp += a[x*width + i]*b[i*second+ y]; } - c[x*width + y] = temp; + c[x*second + y] = temp; } template @@ -27,8 +30,8 @@ void dotGPU(vector &res, const Matrix *lhs, const Matrix &rhs, int star // Copy to CUDA memory - T* h_A = lhs->val.data(); - T* h_B = rhs.val.data(); + const T* h_A = lhs->val.data(); + const T* h_B = rhs.val.data(); T* h_C = res.data(); T *d_a, *d_b, *d_c; @@ -40,8 +43,16 @@ void dotGPU(vector &res, const Matrix *lhs, const Matrix &rhs, int star cudaMemcpy((void *)d_a, h_A + start, sizeof(T)*row1*col1, cudaMemcpyHostToDevice); cudaMemcpy((void *)d_b, h_B, sizeof(T)*row2*col2, cudaMemcpyHostToDevice); - mm<<>>(d_a,d_b,d_c,col1); + mm<<>>(d_a,d_b,d_c,col1,col2); // non blocking function // Copy back from cuda memory - cudaMemcpy(h_C+startRes, (void **)d_c, sizeof(T)*row1*col2, cudaMemcpyDeviceToHost); + cudaMemcpy(h_C+startRes, (void **)d_c, sizeof(T)*row1*col2, cudaMemcpyDeviceToHost); // waits for kernel to get over + + // Clean Up + cudaFree(d_a); + cudaFree(d_b); + cudaFree(d_c); } + +#endif + diff --git a/gpu/impl.h b/gpu/impl.h new file mode 100644 index 0000000..44e3668 --- /dev/null +++ b/gpu/impl.h @@ -0,0 +1 @@ +#include "gpu/dot/impl.cuh" \ No newline at end of file diff --git a/layers/dense.h b/layers/dense.h index e3b35a1..b323175 100644 --- a/layers/dense.h +++ b/layers/dense.h @@ -37,6 +37,9 @@ class Dense{ if(init == GLOROT) { return utils::glorotInit(fan_in, fan_out); } + + // Default return zero vector + return vector(fan_in*fan_out,0); } public: diff --git a/main.cu b/main.cu new file mode 100644 index 0000000..ef148ce --- /dev/null +++ b/main.cu @@ -0,0 +1,54 @@ +#include "buildTensorflowGpu.h" + +// Example of training a network on the buildTensorflow framework. +int main() { + // Load Dataset + Celsius2Faranheit dataset; + dataset.create(5); + + // Create Model + Dense fc1(1,1,NO_ACTIVATION); + + // Initialise Optimiser + SGD sgd(0.01); + + // Train + cout<<"Training started"<({i.first}, {1,1}); + auto tar = new Tensor({i.second}, {1,1}); + + // Forward Prop + auto out = fc1.forward(inp); + + // Get Loss + auto l = new Tensor({-1}, {1,1}); + auto k = tensorOps::multiply(l,tar); + auto loss = tensorOps::add(out,k); // error in loss + auto finalLoss = tensorOps::power(loss,(float)2); + + // Compute backProp + finalLoss->backward(); + // cout<val<({cel}, {1,1}); + auto out1 = fc1.forward(test); + + cout<<"The conversion of "<val< elemsEncounteredPerDim; - // Check whether GPU is accessible or not - bool gpu = false; + // Verifies that the shape provided and val vector provided are compatible in size bool verifyShape(const vector &val, const vector &shape) {