added missing layers

README.md

@@ -8,7 +8,7 @@ The API is clear and simple, the goal is that every C++ developer can contribute
 BeeDNN can run on small devices. It is even possible to learn directly on the device.
 
 Please see at: https://github.com/edeforas/BeeDNN/issues for contributing ideas.
-No dependencies needed, every algorithm rewritten in C++ from scratch.
+No dependencies needed, every algorithm rewritten in C++ from scratch, (and now some using the help of AI agents).
 To increase speed, ones can choose the Eigen library (http://eigen.tuxfamily.org), instead of the internal matrix library.
 
 Initializers:
@@ -18,8 +18,10 @@ Initializers:
 - Zeros, Ones
 
 Layers:
-- Dense (with bias), Dot (without bias)
+- Dense, Dot
+- DotMAM, DenseMAM
 - GlobalGain, GlobalBias, GlobalAffine, Gain, Bias, Affine
+- BatchNormalization
 - Softmax, Softmin
 - PRelu, RRelu, PELU, TERELU, CRelu
 - Gated activations: GLU, ReGLU, Bilinear, SwiGLU, GEGLU, GTU, SeGLU
@@ -29,10 +31,14 @@ Time series:
 - TimeDistributedBias
 - TimeDistributedDot
 - TimeDistributedDense
-- WIP
+- SimpleRNN
+- SimplestRNN
+- GRU
+- LSTM
 
 2D layers:
 - Convolution2D, ChannelBias
+- DepthwiseConvolution2D
 - MaxPool2D, GlobalMaxPool2D, 
 - AveragePooling2D, GlobalAveragePooling2D
 - ZeroPadding2D

python/beednn/BeeDNNUtils.py

@@ -0,0 +1,22 @@
+import numpy as np
+import json
+from . import Model,Layer
+
+def load_from_json(filename):
+    m=Model.Model()
+
+    f=open(filename)
+    j=json.load(f)
+    f.close()
+
+    for a in j:
+        if 'layer' in str(a):
+            l=j[a]
+            if l['name']=='dense':
+                lb=Layer.LayerDense()
+                w_0=l['weight_0']
+                w_1=l['weight_1']
+                activation=l['activation']
+#todo
+            m.append(lb)
+    return m

python/sample_TF_SimpleRNN.py

@@ -2,7 +2,7 @@
 import numpy as np
 import matplotlib.pyplot as plt
 
-from beednn import TFUtils
+from beednn import TFUtils,BeeDNNUtils
 
 #create datas (load, reshape and normalize)
 sunspot="sunspot.txt"
@@ -29,14 +29,19 @@
 plt.title('mse loss')
 plt.grid()
 
-y_pred = model.predict(x_train)
-
+y_pred_tf = model.predict(x_train)
+
+#save for inspection
+TFUtils.save_tf_to_json(model,'saved_model_simplernn.json')
+
+#reload for BeeDNN
+model_beednn=BeeDNNUtils.load_from_json('saved_model_simplernn.json')
+y_pred_beednn=model_beednn.predict(x_train)
+
 plt.figure()
 plt.plot(y_train.flatten()*max_spot,label='y_train')
-plt.plot(y_pred.flatten()*max_spot,label='y_pred')
+plt.plot(y_pred_tf.flatten()*max_spot,label='y_pred_TF')
+plt.plot(y_pred_beednn.flatten()*max_spot,label='y_pred_BeeDNN')
 plt.legend()
 plt.title('truth vs. pred')
-plt.show()
-
-#save for inspection
-TFUtils.save_tf_to_json(model,'saved_model_simplernn.json')
+plt.show()

python/sample_TF_XOR.py

@@ -1,7 +1,7 @@
 import tensorflow as tf
 import numpy as np
 import matplotlib.pyplot as plt
-from beednn import TFUtils
+from beednn import TFUtils,BeeDNNUtils
 
 #create XOR data
 x_train=np.array([[0,0],[0,1],[1, 0],[1,1]])
@@ -15,7 +15,7 @@
 
 #train
 model.compile(loss='mse', optimizer='adam')
-history=model.fit(x_train, y_train, epochs=600)
+history=model.fit(x_train, y_train, epochs=60)
 loss_train=history.history["loss"]
 model.summary()
 
@@ -24,14 +24,20 @@
 plt.title('mse loss')
 plt.grid()
 
-y_pred = model.predict(x_train)
+#save for inspection
+TFUtils.save_tf_to_json(model,'saved_model_xor.json')
+
+#reload for BeeDNN
+model_beednn=BeeDNNUtils.load_from_json('saved_model_xor.json')
+
+y_pred_tf = model.predict(x_train)
+y_pred_beednn=model_beednn.predict(x_train)
+
 plt.figure()
 plt.plot(y_train.flatten(),label='y_train')
-plt.plot(y_pred.flatten(),label='y_pred')
+plt.plot(y_pred_tf.flatten(),label='y_pred_tf')
+plt.plot(y_pred_beednn.flatten(),label='y_pred_beednn')
 plt.legend()
 plt.grid()
 plt.title('truth vs. pred')
-plt.show()
-
-#save for inspection
-TFUtils.save_tf_to_json(model,'saved_model_xor.json')
+plt.show()

samples/sample_classification_MNIST_MAM.cpp

@@ -0,0 +1,107 @@
+// simple MNIST classification with a dense layer, similar as :
+// https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/_index.ipynb
+// validation accuracy > 98.1%, after 20 epochs (1s by epochs)
+
+#include <iostream>
+#include <chrono>
+
+#include "Net.h"
+#include "NetTrain.h"
+#include "MNISTReader.h"
+#include "Metrics.h"
+
+#include "LayerActivation.h"
+#include "LayerDotMAM.h"
+#include "LayerDot.h"
+#include "LayerMaxPool2D.h"
+#include "LayerBias.h"
+#include "LayerDropout.h"
+#include "LayerSoftmax.h"
+
+using namespace std;
+using namespace beednn;
+
+Net model;
+NetTrain netTrain;
+int iEpoch;
+chrono::steady_clock::time_point start;
+
+//////////////////////////////////////////////////////////////////////////////
+void epoch_callback()
+{
+	//compute epoch time
+	chrono::steady_clock::time_point next = chrono::steady_clock::now();
+	auto delta = chrono::duration_cast<std::chrono::milliseconds>(next - start).count();
+	start = next;
+
+    iEpoch++;
+    cout << "Epoch: " << iEpoch << " duration: " << delta << " ms" << endl;
+	cout << "TrainLoss: " << netTrain.get_current_train_loss() << " TrainAccuracy: " << netTrain.get_current_train_accuracy() << " %" ;
+	cout << " ValidationAccuracy: " << netTrain.get_current_validation_accuracy() << " %" << endl;
+
+	cout << endl;
+}
+//////////////////////////////////////////////////////////////////////////////
+int main()
+{
+	cout << "simple classification MNIST with a MAM + dot layer" << endl;
+	cout << "validation accuracy > tbd%, after 15 epochs (1s by epochs)" << endl;
+
+    iEpoch = 0;
+
+	//load and normalize MNIST data
+    cout << "Loading MNIST database..." << endl;
+    MNISTReader mr;
+    if(!mr.load("."))
+    {
+        cout << "MNIST samples not found, please check the *.ubyte files are in the executable folder" << endl;
+        return -1;
+    }
+
+	//create simple net:
+	model.add(new LayerMaxPool2D(28, 28, 1, 4, 4)); //input rows, input cols,input channels, factor rows, factor cols
+	model.add(new LayerDotMAM(784/16, 8));
+	model.add(new LayerBias());
+	model.add(new LayerActivation("Relu"));
+	model.add(new LayerDropout(0.2f)); //reduce overfitting
+	model.add(new LayerDot(8, 10));
+	model.add(new LayerBias());
+	model.add(new LayerSoftmax());
+
+	//setup train options
+	netTrain.set_epochs(15);
+	netTrain.set_batchsize(64);
+	netTrain.set_loss("SparseCategoricalCrossEntropy");
+	netTrain.set_epoch_callback(epoch_callback); //optional, to show the progress
+	netTrain.set_train_data(mr.train_data(),mr.train_truth());
+	netTrain.set_validation_data(mr.validation_data(), mr.validation_truth()); //optional, not used for training, helps to keep the final best model
+
+	// train net
+	cout << "Training..." << endl << endl;
+	start = chrono::steady_clock::now();
+	netTrain.fit(model);
+
+	// show train results
+	MatrixFloat mClassPredicted;
+	model.predict_classes(mr.train_data(), mClassPredicted);
+	Metrics metricsTrain;
+	metricsTrain.compute(mr.train_truth(), mClassPredicted);
+	cout << "Train accuracy: " << metricsTrain.accuracy() << " %" << endl;
+
+	MatrixFloat mClassVal;
+	model.predict_classes(mr.validation_data(), mClassVal);
+	Metrics metricsVal;
+	metricsVal.compute(mr.validation_truth(), mClassVal);
+	cout << "Validation accuracy: " << metricsVal.accuracy() << " %" << endl;
+	cout << "Validation confusion matrix:" << endl << toString(metricsVal.confusion_matrix()) << endl;
+
+	//testu function
+	if (metricsVal.accuracy() < 98.f)
+	{
+		cout << "Test failed! accuracy=" << metricsVal.accuracy() << endl;
+		return -1;
+	}
+
+	cout << "Test succeded." << endl;
+    return 0;
+}

samples/sample_classification_xor_MAM.cpp

@@ -0,0 +1,68 @@
+// this sample shows how to do a simple classification
+// the usecase is to learn a XOR gate
+
+#include <iostream>
+
+#include "Net.h"
+#include "NetTrain.h"
+
+#include "LayerDotMAM.h"
+#include "LayerDense.h"
+#include "LayerBias.h"
+#include "LayerActivation.h"
+
+using namespace std;
+using namespace beednn;
+
+/////////////////////////////////////////////////////////////////////
+// for testU only
+inline bool is_near(double a,double b, double tolerancis=0.001)
+{
+    return fabs(a-b)<tolerancis;
+}
+void test(bool bTest,const string & sMessage="")
+{
+    if(bTest) return;
+
+    cout << "Test failed: " << sMessage << endl;
+    exit(-1);
+}
+/////////////////////////////////////////////////////////////////////
+
+int main()
+{
+	cout << "This sample shows how to do a simple classification using DotMAM layer" << endl;
+	cout << "The usecase is to learn a XOR gate" << endl;
+	//construct network, 2 input, 1 output
+    Net model;
+    model.add(new LayerDotMAM(2,10));
+	model.add(new LayerBias());
+	model.add(new LayerActivation("Relu"));
+	model.add(new LayerDense(10, 1));
+
+    //set the train data
+    float dSamples[]={ 0,0 , 0,1 , 1,0 , 1,1 };
+    float dTruths[]={ 0 , 1 , 1, 0 };
+    const MatrixFloat mSamples=fromRawBuffer(dSamples,4,2);
+    const MatrixFloat mTruth=fromRawBuffer(dTruths,4,1);
+
+    //optimize network
+	NetTrain netFit;
+	netFit.set_epochs(500);
+	netFit.set_train_data(mSamples, mTruth);
+
+	//predict and show results
+	netFit.fit(model);
+	MatrixFloat mOut;
+	model.predict_classes(mSamples, mOut);
+	cout << "0_xor_0=" << mOut(0) << endl << "0_xor_1=" << mOut(1) << endl << "1_xor_0=" << mOut(2) << endl << "1_xor_1=" << mOut(3) << endl;
+
+	//simple testU code
+	test(is_near(mOut(0),0));
+	test(is_near(mOut(1),1));
+	test(is_near(mOut(2),1));
+	test(is_near(mOut(3),0));
+
+	cout << "Test succeded." << endl;
+    return 0;
+}

src/CMakeLists.txt

@@ -1,5 +1,6 @@
 set(BEEDNN_FILES
 	Activations.cpp Activations.h
+	LayerBatchNormalization.cpp LayerBatchNormalization.h
 	CIFAR10Reader.cpp CIFAR10Reader.h
 	Metrics.cpp Metrics.h
 	CsvFileReader.cpp CsvFileReader.h
@@ -19,7 +20,9 @@ set(BEEDNN_FILES
 	LayerCRelu.cpp LayerCRelu.h
 	LayerDense.cpp LayerDense.h
 	LayerDot.cpp LayerDot.h
+	LayerDotMAM.cpp LayerDotMAM.h
 	LayerDropout.cpp LayerDropout.h
+	LayerDepthwiseConvolution2D.cpp LayerDepthwiseConvolution2D.h
 	LayerFactory.cpp LayerFactory.h
 	LayerGain.cpp LayerGain.h
 	LayerGaussianDropout.cpp LayerGaussianDropout.h
@@ -43,6 +46,8 @@ set(BEEDNN_FILES
 	LayerSeGLU.cpp LayerSeGLU.h
 	LayerSimpleRNN.cpp LayerSimpleRNN.h
 	LayerSimplestRNN.cpp LayerSimplestRNN.h
+	LayerGRU.cpp LayerGRU.h
+	LayerLSTM.cpp LayerLSTM.h
 	LayerSoftmax.cpp LayerSoftmax.h
 	LayerSoftmin.cpp LayerSoftmin.h
 	LayerSwiGLU.cpp LayerSwiGLU.h

src/Layer.h

@@ -29,7 +29,7 @@ class Layer
     virtual void init();
     virtual void backpropagation(const MatrixFloat &mIn,const MatrixFloat &mGradientOut, MatrixFloat &mGradientIn)=0;
 
-	void set_train_mode(bool bTrainMode); //set to true to train, to false to test
+	void set_train_mode(bool bTrainMode); //set to true to train, to false to predict only
 
     void set_weight_initializer(const std::string& _sWeightInitializer);
     std::string weight_initializer() const;