From a5afbf5a1833d4e7a3c35f4ba92a2a8cb0306285 Mon Sep 17 00:00:00 2001 From: Udit Arora Date: Sun, 7 Apr 2019 20:09:42 +0530 Subject: [PATCH 1/3] Added SubtractOperation --- operations/subtractOperation.h | 28 ++++++++++++++++++++++++++ operations/subtractOperation_impl.h | 31 +++++++++++++++++++++++++++++ overloads/tensor.h | 15 ++++++++++++++ overloads/vector.h | 23 +++++++++++++++++++++ types/matrix.h | 9 +++++++++ types/tensor.h | 1 + 6 files changed, 107 insertions(+) create mode 100644 operations/subtractOperation.h create mode 100644 operations/subtractOperation_impl.h diff --git a/operations/subtractOperation.h b/operations/subtractOperation.h new file mode 100644 index 0000000..97dc2d0 --- /dev/null +++ b/operations/subtractOperation.h @@ -0,0 +1,28 @@ +/* + This file defines the SubtractOperation class which represents the subtraction of + two tensors. +*/ + +#include "operations/operation.h" + +#ifndef __OP_SUBTRACT_INCLUDED__ +#define __OP_SUBTRACT_INCLUDED__ + +template +class SubtractOperation : public Operation { + public: + + SubtractOperation(Tensor *t1, Tensor *t2) { + this->t1 = t1; + this->t2 = t2; + } + + void backward(Matrix grad); + + Tensor* forward(); + + Tensor forwardDeprecated(); + +}; + +#endif diff --git a/operations/subtractOperation_impl.h b/operations/subtractOperation_impl.h new file mode 100644 index 0000000..43375f7 --- /dev/null +++ b/operations/subtractOperation_impl.h @@ -0,0 +1,31 @@ +/* + This file contains the implementation of the forward and backward pass of + the subtract operation. +*/ + +#include "operations/subtractOperation.h" + +#ifndef __OP_SUBTRACT_IMPL_INCLUDED__ +#define __OP_SUBTRACT_IMPL_INCLUDED__ + +template +void SubtractOperation::backward(Matrix grad) { + // Distributing case with negative: where one gradients is backproped + // as is, and the other is backproped with a negative sign + this->t1->backward(grad); + this->t2->backward(-1 * grad); +} + +template +Tensor* SubtractOperation::forward() { + this->t3 = new Tensor(this->t1->val - this->t2->val, this); + return this->t3; +} + +template +Tensor SubtractOperation::forwardDeprecated() { + this->t3 = new Tensor(this->t1->val - this->t2->val, this); + return *this->t3; +} + +#endif diff --git a/overloads/tensor.h b/overloads/tensor.h index f61733d..d77b626 100644 --- a/overloads/tensor.h +++ b/overloads/tensor.h @@ -24,6 +24,21 @@ namespace tensorOps { return add(one,two); } + // Subtraction + template + Tensor* subtract(Tensor* one, Tensor* two) { + one->frontOp = new SubtractOperation(one, two); + two->frontOp = one->frontOp; + return one->frontOp->forward(); + } + + // Subtraction with Scalar + template + Tensor* subtract(T v, Tensor* two) { + auto one = new Tensor(vector(two->val.val.size(),v),two->val.shape); + return subtract(one,two); + } + // Divide template Tensor* divide(Tensor* one, Tensor* two) { diff --git a/overloads/vector.h b/overloads/vector.h index 9aa14ba..817f9c5 100644 --- a/overloads/vector.h +++ b/overloads/vector.h @@ -62,6 +62,29 @@ vector operator + (T a, const vector &b) { return arr; } +// Vector Subtraction +template +vector operator - (vector &a, const vector &b) { + assert("Tensors are not of the same size !" && a.size() == b.size()); + vector arr; + for(int i = 0;i +vector operator - (T a, const vector &b) { + vector arr; + for(int i = 0;i vector operator / (vector &a, const vector &b) { diff --git a/types/matrix.h b/types/matrix.h index 6c87200..fe65500 100644 --- a/types/matrix.h +++ b/types/matrix.h @@ -245,6 +245,15 @@ struct Matrix{ return Matrix(res, resShape); } + // Performs elementwise subtraction + Matrix operator - (const Matrix &rhs) { + assert("Shapes aren't compatible for subtraction !" && + verifyShapeForElementwiseOperation(this->shape, rhs.shape)); + + auto res = this->val - rhs.val; + auto resShape = this->shape; + return Matrix(res, resShape); + } // Performs elementwise division Matrix operator / (const Matrix &rhs) { diff --git a/types/tensor.h b/types/tensor.h index bae4ec1..aeafb0f 100644 --- a/types/tensor.h +++ b/types/tensor.h @@ -20,6 +20,7 @@ #include "operations/operation.h" #include "operations/addOperation.h" +#include "operations/subtractOperation.h" #include "operations/multiplyOperation.h" #include "operations/divideOperation.h" #include "operations/exponentOperation.h" From ad21f32094f10fcfcd70ca4006c3b6b8cd34264c Mon Sep 17 00:00:00 2001 From: Udit Arora Date: Thu, 11 Apr 2019 14:25:07 +0530 Subject: [PATCH 2/3] Added AverageOperation for taking the average of multiple tensors --- operations/averageOperation.h | 29 ++++++++++++++ operations/averageOperation_Impl.h | 64 ++++++++++++++++++++++++++++++ overloads/tensor.h | 15 +++++++ types/tensor.h | 1 + 4 files changed, 109 insertions(+) create mode 100644 operations/averageOperation.h create mode 100644 operations/averageOperation_Impl.h diff --git a/operations/averageOperation.h b/operations/averageOperation.h new file mode 100644 index 0000000..a8c0bf5 --- /dev/null +++ b/operations/averageOperation.h @@ -0,0 +1,29 @@ +/* + This file defines the AverageOperation class which represents the average of + multiple tensors. +*/ + +#include "operations/operation.h" + +#ifndef __OP_AVG_INCLUDED__ +#define __OP_AVG_INCLUDED__ + +template +class AverageOperation : public Operation { + public: + + vector> tensors; + + AverageOperation(vector*>& tensors) { + this->tensors = tensors; + } + + void backward(Matrix grad); + + Tensor forwardDeprecated(); + + Tensor* forward(); + +}; + +#endif diff --git a/operations/averageOperation_Impl.h b/operations/averageOperation_Impl.h new file mode 100644 index 0000000..51712d3 --- /dev/null +++ b/operations/averageOperation_Impl.h @@ -0,0 +1,64 @@ +/* + This file contains the implementation of the forward and backward pass of + the average operation. +*/ + +#include "operations/averageOperation.h" + +#ifndef __OP_AVG_IMPL_INCLUDED__ +#define __OP_AVG_IMPL_INCLUDED__ + +/* + Backpropogation of the average operation. The average operation distributes the gradient. So it + effectively just transfers the gradient coming in to the various input sources after scaling + it by the number of inputs. +*/ +template +void AverageOperation::backward(Matrix grad) { + if (tensors.size() == 0) { // To avoid division by zero + return; + } + + auto scaledGrad = grad / tensors.size(); + for(auto t : tensors) { + t->backward(scaledGrad); + } +} + +/* + Forward Propogation of the average operation. Returns a tensor + + TODO: Remove: See average operation impl for more details +*/ +template +Tensor AverageOperation::forwardDeprecated() { + return NULL; +} + +/* + Forward Propogation of the operation. Return pointer to the tensor. +*/ +template +Tensor* AverageOperation::forward() { + if (tensors.size() == 0) { // To avoid division by zero + return NULL; + } + + Matrix sum = NULL; + + for(auto t : tensors) { + if(sum == NULL) { + sum = t->val; + } + else { + sum += t->val; + } + } + + sum /= tensors.size(); + + this->t3 = new Tensor(sum, this); + return this->t3; +} + +#endif diff --git a/overloads/tensor.h b/overloads/tensor.h index d77b626..8c0bce4 100644 --- a/overloads/tensor.h +++ b/overloads/tensor.h @@ -91,6 +91,21 @@ namespace tensorOps { return one->frontOp->forward(); } + // Average + template + Tensor* average(vector*>& tensors) { + if (tensors.size() == 0) { // To avoid division by zero. Should we do this? + return NULL; + } + + Operation op = new AverageOperation(tensors); + for (auto t : tensors) { + t->frontOp = op; + } + + return tensors[0]->frontOp->forward(); + } + }; #endif diff --git a/types/tensor.h b/types/tensor.h index aeafb0f..113ae64 100644 --- a/types/tensor.h +++ b/types/tensor.h @@ -26,6 +26,7 @@ #include "operations/exponentOperation.h" #include "operations/dotOperation.h" #include "operations/sigmoidOperation.h" +#include "operations/averageOperation.h" #ifndef __TENSOR_FLOAT_INCLUDED__ #define __TENSOR_FLOAT_INCLUDED__ From 0bfd1962215c0014ef2242c246da73e17eef30b3 Mon Sep 17 00:00:00 2001 From: Udit Arora Date: Thu, 11 Apr 2019 14:32:09 +0530 Subject: [PATCH 3/3] Added more overloads in tensor.h for reverse argument order of vector and scalar --- overloads/tensor.h | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/overloads/tensor.h b/overloads/tensor.h index 8c0bce4..49d6f38 100644 --- a/overloads/tensor.h +++ b/overloads/tensor.h @@ -17,13 +17,20 @@ namespace tensorOps { return one->frontOp->forward(); } - // Addition with Scalar + // Addition with Scalar - Scalar first template Tensor* add(T v, Tensor* two) { auto one = new Tensor(vector(two->val.val.size(), v), two->val.shape); return add(one,two); } + // Addition with Scalar - Vector first + template + Tensor* add(Tensor* two, T v) { + auto one = new Tensor(vector(two->val.val.size(), v), two->val.shape); + return add(one,two); + } + // Subtraction template Tensor* subtract(Tensor* one, Tensor* two) { @@ -32,13 +39,20 @@ namespace tensorOps { return one->frontOp->forward(); } - // Subtraction with Scalar + // Subtraction with Scalar - Scalar first template Tensor* subtract(T v, Tensor* two) { auto one = new Tensor(vector(two->val.val.size(),v),two->val.shape); return subtract(one,two); } + // Subtraction with Scalar - Vector first + template + Tensor* subtract(Tensor* two, T v) { + auto one = new Tensor(vector(two->val.val.size(),v),two->val.shape); + return subtract(one,two); + } + // Divide template Tensor* divide(Tensor* one, Tensor* two) { @@ -47,13 +61,20 @@ namespace tensorOps { return one->frontOp->forward(); } - // Divide Scalar + // Divide Scalar - Scalar first template Tensor* divide(T v, Tensor* two) { auto one = new Tensor(vector(two->val.val.size(), v), two->val.shape); return divide(one,two); } + // Divide Scalar - Vector first + template + Tensor* divide(Tensor* two, T v) { + auto one = new Tensor(vector(two->val.val.size(), v), two->val.shape); + return divide(one,two); + } + // Multiply template Tensor* multiply(Tensor* one, Tensor* two) { @@ -62,13 +83,20 @@ namespace tensorOps { return one->frontOp->forward(); } - // Multiply with scalar + // Multiply with scalar - Scalar first template Tensor* multiply(T v, Tensor* two) { auto one = new Tensor(vector(two->val.val.size(), v), two->val.shape); return multiply(one,two); } + // Multiply with scalar - Vector first + template + Tensor* multiply(Tensor* two, T v) { + auto one = new Tensor(vector(two->val.val.size(), v), two->val.shape); + return multiply(one,two); + } + // Dot Product template Tensor* dot(Tensor* one, Tensor* two) {