diff --git a/.gitattributes b/.gitattributes index 9faefd8..27af723 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1 @@ .csv filter=lfs diff=lfs merge=lfs -text -resources/mnist/mnist_train.csv filter=lfs diff=lfs merge=lfs -text diff --git a/nnf/activations/__init__.py b/nnf/activations/__init__.py index bc61376..7dfb054 100644 --- a/nnf/activations/__init__.py +++ b/nnf/activations/__init__.py @@ -3,5 +3,7 @@ from nnf.activations.softmax import Softmax from nnf.activations.sigmoid import Sigmoid from nnf.activations.leaky_relu import LeakyReLU +from nnf.activations.tanh import Tanh -__all__ = ['Activation', 'ReLU', 'Softmax', 'Sigmoid', 'LeakyReLU'] \ No newline at end of file + +__all__ = ['Activation', 'ReLU', 'Softmax', 'Sigmoid', 'LeakyReLU', 'Tanh'] \ No newline at end of file diff --git a/nnf/activations/base.py b/nnf/activations/base.py index 2780ac6..e6042e8 100644 --- a/nnf/activations/base.py +++ b/nnf/activations/base.py @@ -13,4 +13,7 @@ def forward(self, inputs): raise NotImplementedError def backward(self, dvalues): - raise NotImplementedError \ No newline at end of file + raise NotImplementedError + + def get_params(self): + return super().get_params() \ No newline at end of file diff --git a/nnf/activations/leaky_relu.py b/nnf/activations/leaky_relu.py index 6837577..718cda6 100644 --- a/nnf/activations/leaky_relu.py +++ b/nnf/activations/leaky_relu.py @@ -1,4 +1,6 @@ import numpy as np +from typing import override, Dict + from nnf.activations.base import Activation class LeakyReLU(Activation): @@ -50,3 +52,18 @@ def backward(self, dvalues): # For inputs <= 0, multiply the gradient by alpha self.dinputs[self.inputs <= 0] *= self.alpha return self.dinputs + + @override + def get_params(self): + return { + "type" : "LeakyReLU", + "attrs" : { + "alpha": self.alpha + } + } + + @override + def set_params(self, params : Dict): + for key, val in params.items(): + setattr(self, key, val) + \ No newline at end of file diff --git a/nnf/activations/relu.py b/nnf/activations/relu.py index ffc93a7..22138fd 100644 --- a/nnf/activations/relu.py +++ b/nnf/activations/relu.py @@ -1,4 +1,6 @@ import numpy as np +from typing import override + from nnf.activations.base import Activation class ReLU(Activation): @@ -25,4 +27,11 @@ def backward(self, dvalues): self.dinputs = dvalues.copy() self.dinputs[self.inputs <= 0] = 0 - return self.dinputs \ No newline at end of file + return self.dinputs + + @override + def get_params(self): + return { + "type" : "ReLU", + "attrs" : {} + } \ No newline at end of file diff --git a/nnf/activations/sigmoid.py b/nnf/activations/sigmoid.py index cee6d4d..478202a 100644 --- a/nnf/activations/sigmoid.py +++ b/nnf/activations/sigmoid.py @@ -1,4 +1,6 @@ import numpy as np +from typing import override + from nnf.activations.base import Activation class Sigmoid(Activation): @@ -26,4 +28,11 @@ def backward(self, dvalues): # derivative of the sigmoid: f'(x) = f(x) * (1 - f(x)) self.dinputs = dvalues * (self.output * (1 - self.output)) - return self.dinputs \ No newline at end of file + return self.dinputs + + @override + def get_params(self): + return { + "type" : "Sigmoid", + "attrs" : {} + } \ No newline at end of file diff --git a/nnf/activations/softmax.py b/nnf/activations/softmax.py index be6ddfd..06f331b 100644 --- a/nnf/activations/softmax.py +++ b/nnf/activations/softmax.py @@ -33,6 +33,8 @@ """ import numpy as np +from typing import override + from nnf.activations.base import Activation class Softmax(Activation): @@ -84,3 +86,10 @@ def backward(self, dvalues): """ self.dinputs = dvalues # usually combined with loss return self.dinputs + + @override + def get_params(self): + return { + "type" : "Softmax", + "attrs" : {} + } \ No newline at end of file diff --git a/nnf/activations/tanh.py b/nnf/activations/tanh.py index 2bef3ea..52bab07 100644 --- a/nnf/activations/tanh.py +++ b/nnf/activations/tanh.py @@ -31,6 +31,8 @@ """ import numpy as np +from typing import override + from nnf.activations.base import Activation class Tanh(Activation): @@ -70,3 +72,10 @@ def backward(self, dvalues): """ self.dinputs = dvalues * (1 - self.output ** 2) return self.dinputs + + @override + def get_params(self): + return { + "type" : "Tanh", + "attrs" : {} + } \ No newline at end of file diff --git a/nnf/layers/base.py b/nnf/layers/base.py index e3156e7..cee4dee 100644 --- a/nnf/layers/base.py +++ b/nnf/layers/base.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from typing import Dict class Layer(ABC): def __init__(self): @@ -19,4 +20,10 @@ def forward(self, inputs): @abstractmethod def backward(self, dvalues): + pass + + def get_params(self): + return {} + + def set_params(self, params : Dict): pass \ No newline at end of file diff --git a/nnf/layers/dense.py b/nnf/layers/dense.py index 14f8e2e..3556c70 100644 --- a/nnf/layers/dense.py +++ b/nnf/layers/dense.py @@ -1,4 +1,6 @@ import numpy as np +from typing import override, Dict, List + from nnf.layers.base import Layer class Dense(Layer): @@ -8,12 +10,12 @@ class Dense(Layer): -------------------------------------------- """ - def __init__(self, n_inputs, n_neurons): + def __init__(self, n_inputs = 1, n_neurons = 1): """ The function initializes weights with random values and biases with zeros. """ super().__init__() - + self.n_inputs = n_inputs self.n_neurons = n_neurons @@ -30,6 +32,7 @@ def __init__(self, n_inputs, n_neurons): self.dbiases = None # Parameters + self.params = self.weights.size + self.biases.size def forward(self, inputs): @@ -52,4 +55,31 @@ def backward(self, dvalues): self.dbiases = np.sum(dvalues, axis=0, keepdims=True) self.dinputs = np.dot(dvalues, self.weights.T) - return self.dinputs \ No newline at end of file + return self.dinputs + + @override + def get_params(self): + return { + "type" : "Dense", + "attrs" : { + "n_inputs" : self.n_inputs, + "n_neurons" : self.n_neurons, + "trainable" : self.trainable, + "weights" : self.weights, + "biases" : self.biases, + "dweights" : self.dweights, + "dbiases" : self.dbiases + } + } + + @override + def set_params(self, params : Dict): + for key, val in params.items(): + # If the key is one of the specified attributes (e.g., "n_inputs", "n_neurons", etc.) + # and the value is a list, convert the value to a NumPy array and set it as an attribute. + # Basically, to how they were originally! + if key in ("n_inputs", "n_neurons", "weights", "biases", "dweights", "dbiases") and type(key) == List: + setattr(self, key, np.array(val)) + else: + setattr(self, key, val) + \ No newline at end of file diff --git a/nnf/models/model.py b/nnf/models/model.py index 9a51664..dd629eb 100644 --- a/nnf/models/model.py +++ b/nnf/models/model.py @@ -1,6 +1,9 @@ import math import numpy as np -from typing import List +from typing import List, Dict +import os +from datetime import datetime +import json from tqdm import tqdm from tabulate import tabulate @@ -8,8 +11,8 @@ from nnf.layers.base import Layer from nnf.losses.base import Loss from nnf.optimizers.base import Optimizer +from nnf.utils import LAYER_CLASSES -# Module docstring """ This module defines a neural network Model class that combines layers and handles the training, forward pass, and backward pass operations. @@ -274,12 +277,12 @@ def evaluate(self, X_test, y_test): test_prec = self._calculate_precision(test_output, y_test) * 100 evaluation_summary = [ - ["Training Loss", train_loss], - ["Training Accuracy", train_acc], - ["Training Precision", train_prec], - ["Test Loss", test_loss], - ["Test Loss", test_acc], - ["Test Loss", test_prec], + ["Training Loss" , train_loss], + ["Training Accuracy" , train_acc], + ["Training Precision" , train_prec], + ["Test Loss" , test_loss], + ["Test Accuracy" , test_acc], + ["Test Precision" , test_prec], ] table = tabulate( @@ -369,5 +372,163 @@ def _calculate_precision(self, output, y_true): precision = np.mean(ppc) return precision + + def get_model_attrs(self): + """ + Retrieves the model's attribute values. + + Returns: + dict: A dictionary containing the model's attributes: + - "name": The model's name. + - "loss": The name of the loss function used by the model. + - "optimizer": The model's optimizer parameters. + - "clip_value": The model's clip value. + - "shuffle": The model's shuffle setting. + """ - \ No newline at end of file + return { + "name" : self.name, + "loss" : self.loss.name, + "optimizer" : self.optimizer.get_params(), + "clip_value" : self.clip_value, + "shuffle" : self.shuffle + } + + def set_model_attrs(self, attrs : Dict): + """ + Sets the model's attributes from the provided dictionary. + + Args: + attrs (dict): A dictionary of attribute names and values to set. + If a value is a dictionary with keys "type" and "attrs", + it initializes a layer object from `LAYER_CLASSES` and sets its parameters. + """ + + for key, val in attrs.items(): + if isinstance(val, dict): + obj = LAYER_CLASSES[val["type"]]() + obj.set_params(val["attrs"]) + setattr(self, key, obj) + elif isinstance(val, str) and val in LAYER_CLASSES: + setattr(self, key, LAYER_CLASSES[val]()) + else: + setattr(self, key, val) + + def __default_model_path(self): + """ + Generates the default file path for saving the model. + + Returns: + str: The path where the model will be saved (e.g., "saved_models/model_YYYYMMDD_HHMMSS.json"). + """ + + model_name = None + + os.makedirs("saved_models", exist_ok=True) + + if not self.name: + model_name = datetime.now().strftime("model_%Y%m%d_%H%M%S") + + return f"saved_models/{model_name}.json" + + def save_model(self, *, file_path : str = None): + """ + Saves the model to a JSON file. + + Args: + file_path (str, optional): The path to save the model file. If not provided, a default path will be used. + + """ + + # Converts attributes to formats suitable for JSON serialization + # (e.g., converting numpy arrays to Python lists) + def convert_types(obj): + if isinstance(obj, np.ndarray): + return obj.tolist() + + return obj + + model_attrs = [] + layer_attrs = [] + + if not file_path: + file_path = self.__default_model_path() + + for layer in self.layers: + attr = layer.get_params() + + layer_attrs.append({ + "type": attr["type"], + "attrs": {k: convert_types(v) for k, v in attr.get("attrs", {}).items()} + }) + + model_attrs = self.get_model_attrs() + + data = [ + { + "model": model_attrs, + "layers": layer_attrs + } + ] + + with open(file_path, "w") as f: + json.dump(data, f, indent=2) + + print(f"Model saved to {file_path}") + + @staticmethod + def load_model(file_path : str): + """ + Loads a model from a JSON file. + + Args: + file_path (str): The path to the model file to load. + + Returns: + Model: The loaded model object. + + Raises: + FileNotFoundError: If the model file is not found at the given file path. + json.JSONDecodeError: If there is an error decoding the JSON file. + PermissionError: If there is no permission to read the specified file. + """ + + try: + with open(file_path, "r") as f: + data = json.load(f) + + data = data[0] + + model_attrs = data["model"] + layer_attrs = data["layers"] + + layers = [] + for layer in layer_attrs: + layer_type = layer["type"] + + attrs = layer["attrs"] if layer["attrs"] else {} + + obj : Layer = LAYER_CLASSES[layer_type]() + obj.set_params(attrs) + + layers.append(obj) + + model_attrs = { + key: LAYER_CLASSES[val]() if isinstance(val, str) and val in LAYER_CLASSES else val + for key, val in model_attrs.items() + } + + model = Model(*layers) + model.set_model_attrs(model_attrs) + + return model + + except FileNotFoundError as e: + raise FileNotFoundError(f"Failed to load model. The model at '{file_path}' could not be found! - {e}") + + except json.JSONDecodeError as e: + raise json.JSONDecodeError(f"Error: Invalid JSON - {e}", e.doc, e.pos) + + except PermissionError: + raise PermissionError(f"Error loading Model. You don't have permission to read '{file_path}'") + \ No newline at end of file diff --git a/nnf/optimizers/base.py b/nnf/optimizers/base.py index 0217668..7018b37 100644 --- a/nnf/optimizers/base.py +++ b/nnf/optimizers/base.py @@ -1,5 +1,7 @@ from nnf.layers.base import Layer +from typing import Dict + class Optimizer: """ ------------------------ @@ -15,7 +17,6 @@ def __init__(self, learning_rate : int = 0.1, decay = 0.0): self.name = self.__class__.__name__ - def pre_update_params(self): """ The function `pre_update_params` adjusts the current learning rate based on a decay factor and @@ -36,4 +37,11 @@ def post_update_params(self): Method called after updating params. """ - self.iterations += 1 \ No newline at end of file + self.iterations += 1 + + def get_params(self): + raise NotImplementedError("Subclasses must implement update_params()") + + def set_params(self, params : Dict): + raise NotImplementedError("Subclasses must implement update_params()") + \ No newline at end of file diff --git a/nnf/optimizers/gradient_descent.py b/nnf/optimizers/gradient_descent.py index 2dbb051..71437b0 100644 --- a/nnf/optimizers/gradient_descent.py +++ b/nnf/optimizers/gradient_descent.py @@ -1,3 +1,5 @@ +from typing import Dict, override + from nnf.optimizers.base import Optimizer from nnf.layers.base import Layer @@ -28,3 +30,18 @@ def update_params(self, layer : Layer): def pre_update_params(self): self.iterations += 1 self.current_learning_rate = self.learning_rate / (1.0 + self.decay * self.iterations) + + @override + def get_params(self): + return { + "type" : "GradientDescent", + "attrs" : { + "learning_rate" : self.learning_rate, + "decay" : self.decay + } + } + + @override + def set_params(self, params : Dict): + for key, val in params.items(): + setattr(self, key, val) \ No newline at end of file diff --git a/nnf/optimizers/momentum.py b/nnf/optimizers/momentum.py index 3e1aa1e..8edd584 100644 --- a/nnf/optimizers/momentum.py +++ b/nnf/optimizers/momentum.py @@ -1,4 +1,6 @@ import numpy as np +from typing import Dict, override + from nnf.optimizers.base import Optimizer from nnf.layers.base import Layer @@ -63,3 +65,19 @@ def update_params(self, layer: Layer): ) layer.biases -= self.learning_rate * bias_velocity self.velocities[layer]["biases"] = bias_velocity + + @override + def get_params(self): + return { + "type" : "Momentum", + "attrs" : { + "learning_rate" : self.learning_rate, + "decay" : self.decay, + "momentum" : self.momentum + } + } + + @override + def set_params(self, params : Dict): + for key, val in params.items(): + setattr(self, key, val) \ No newline at end of file diff --git a/nnf/utils/__init__.py b/nnf/utils/__init__.py new file mode 100644 index 0000000..c9ab817 --- /dev/null +++ b/nnf/utils/__init__.py @@ -0,0 +1,3 @@ +from nnf.utils.layer_config import LAYER_CLASSES + +__all__ = ['LAYER_CLASSES'] \ No newline at end of file diff --git a/nnf/utils/layer_config.py b/nnf/utils/layer_config.py new file mode 100644 index 0000000..fafbf5a --- /dev/null +++ b/nnf/utils/layer_config.py @@ -0,0 +1,28 @@ +from nnf.activations import LeakyReLU +from nnf.activations import ReLU +from nnf.activations import Sigmoid +from nnf.activations import Softmax +from nnf.activations import Tanh + +from nnf.layers import Dense + +from nnf.losses import MSE +from nnf.losses import BinaryCrossEntropy +from nnf.losses import CategoricalCrossEntropy + +from nnf.optimizers import GradientDescent +from nnf.optimizers import Momentum + +LAYER_CLASSES = { + "Dense" : Dense, + "ReLU" : ReLU, + "LeakyRelU" : LeakyReLU, + "Sigmoid" : Sigmoid, + "Softmax" : Softmax, + "Tanh" : Tanh, + "GradientDescent" : GradientDescent, + "Momentum" : Momentum, + "MSE" : MSE, + "BinaryCrossEntropy" : BinaryCrossEntropy, + "CategoricalCrossEntropy" : CategoricalCrossEntropy +} \ No newline at end of file diff --git a/tests/test_model/test_model.py b/tests/test_model/test_model.py index 575cf89..e09d7c4 100644 --- a/tests/test_model/test_model.py +++ b/tests/test_model/test_model.py @@ -1,5 +1,9 @@ import numpy as np import pytest +import tempfile +import os +import json + from nnf.layers.dense import Dense from nnf.losses import MSE from nnf.optimizers.gradient_descent import GradientDescent @@ -23,6 +27,9 @@ def simple_model(): loss = MSE() optimizer = GradientDescent(learning_rate=0.01) model.set(loss, optimizer) + model.name = "TestModel" + model.clip_value = 1.0 + model.shuffle = False return model def test_train_and_predict(mock_data, simple_model): @@ -39,23 +46,74 @@ def test_train_and_predict(mock_data, simple_model): final_loss = model.loss.calculate(predictions, y) assert final_loss <= initial_loss, "Model did not reduce the loss during training" -# def test_model_summary(simple_model): -# model = simple_model +def test_get_model_attrs(simple_model): + attrs = simple_model.get_model_attrs() + + assert isinstance(attrs, dict) + assert attrs["name"] == "TestModel" + assert attrs["loss"] == "MSE" + assert isinstance(attrs["optimizer"], dict) + assert attrs["clip_value"] == 1.0 + assert attrs["shuffle"] is False -# # Capture the output of the summary -# from io import StringIO -# import sys +def test_set_model_attrs_preserves_values(simple_model): + attrs = simple_model.get_model_attrs() + model = Model() + model.set_model_attrs(attrs) + + assert model.name == "TestModel" + assert model.loss.name == "MSE" + assert isinstance(model.loss, MSE) + assert model.optimizer.get_params() == attrs["optimizer"] + assert model.clip_value == 1.0 + assert model.shuffle is False -# # Redirect stdout to capture print output -# captured_output = StringIO() -# sys.stdout = captured_output +def test_save_and_load_model(simple_model): + with tempfile.NamedTemporaryFile(delete=False, suffix=".json") as temp_file: + path = temp_file.name -# # Call the summary method -# model.summary() + try: + simple_model.save_model(file_path=path) + assert os.path.exists(path), "Model file was not created" -# # Check if the summary includes expected information -# assert "Total Layers: 2" in captured_output.getvalue(), "Model summary does not include total layers" -# assert "Total parameters" in captured_output.getvalue(), "Model summary does not include total parameters" - -# # Reset redirect. -# sys.stdout = sys.__stdout__ + with open(path) as f: + saved = json.load(f) + assert isinstance(saved, list) and "model" in saved[0] + + loaded_model = Model.load_model(path) + + assert isinstance(loaded_model, Model) + assert loaded_model.name == simple_model.name + assert loaded_model.loss.name == simple_model.loss.name + assert loaded_model.optimizer.get_params() == simple_model.optimizer.get_params() + assert loaded_model.clip_value == simple_model.clip_value + assert loaded_model.shuffle == simple_model.shuffle + assert len(loaded_model.layers) == len(simple_model.layers) + finally: + os.remove(path) + +def test_default_model_path_creates_file(simple_model): + # Force no file_path + simple_model.name = None + + simple_model.save_model() # Should trigger default path creation + + saved_dir = "saved_models" + files = os.listdir(saved_dir) + model_files = [f for f in files if f.startswith("model_") and f.endswith(".json")] + + assert len(model_files) > 0, "Default model file was not created" + + for f in model_files: + os.remove(os.path.join(saved_dir, f)) + +def test_load_model_file_not_found(): + with pytest.raises(FileNotFoundError): + Model.load_model("non_existent_model.json") + +def test_load_model_invalid_json(tmp_path): + bad_file = tmp_path / "invalid.json" + bad_file.write_text("{not valid json}") + + with pytest.raises(json.JSONDecodeError): + Model.load_model(str(bad_file)) \ No newline at end of file