From 11afbb636880c71eaa4538e3fff2f846acc76eec Mon Sep 17 00:00:00 2001 From: Kibrewossen <47031138+kebtes@users.noreply.github.com> Date: Sat, 19 Apr 2025 22:08:37 +0300 Subject: [PATCH 1/3] Add evaluate method and metrics calculations to Model class --- nnf/models/model.py | 156 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 154 insertions(+), 2 deletions(-) diff --git a/nnf/models/model.py b/nnf/models/model.py index 0b48083..9a51664 100644 --- a/nnf/models/model.py +++ b/nnf/models/model.py @@ -39,6 +39,10 @@ def __init__(self, *layers: Layer, name: str = None): self.clip_value = 1.0 self.shuffle = False + # Store training data internally for later evaluation (used in model.evaluate) + self._X_train = None + self._y_train = None + def set(self, loss: Loss, optimizer: Optimizer): """ Set the loss function and optimizer for the model. @@ -88,6 +92,10 @@ def train(self, X, y, *, epochs=1, batch_size: int = None): epochs (int): Number of epochs to train for. batch_size (int): Size of the training batches. Defaults to None. """ + + self._X_train = X + self._y_train = y + if batch_size is None: batch_size = len(X) @@ -143,7 +151,9 @@ def predict(self, X): Returns: Predictions from the model. """ - return self.forward(X) + + self._predictions = self.forward(X) + return self._predictions def summary(self): """ @@ -218,4 +228,146 @@ def _print_summary(self, header: List, model_summary: List, total_params: int, p # Print additional information such as total layers, total parameters, loss function, and shapes print(f"\nTotal Layers: {len(self.layers)}") print(f"Total parameters: {total_params:,}") # Formatting the total parameters with commas - print(f"Loss: {self.loss.name}") \ No newline at end of file + print(f"Loss: {self.loss.name}") + + def evaluate(self, X_test, y_test): + """ + Evaluate the model on both training and test data. + + This method performs a forward pass using the model's training data + and the provided test data, calculates the loss, accuracy, and precision + for each, and displays the results in a formatted table. + + The evaluation metrics include: + - Training Loss + - Training Accuracy (percentage) + - Training Precision (percentage) + - Test Loss + - Test Accuracy (percentage) + - Test Precision (percentage) + + Parameters: + X_test (ndarray): Input features for the test dataset. + y_test (ndarray): True labels for the test dataset. + + Returns: + dict: A dictionary containing all evaluation metrics: + { + "train_loss": float, + "train_acc": float, + "train_precision": float, + "test_loss": float, + "test_acc": float, + "test_precision": float + } + """ + # Training + train_output = self.forward(self._X_train) + train_loss = self.loss.calculate(train_output, self._y_train) + train_acc = self._calculate_accuracy(train_output, self._y_train) * 100 + train_prec = self._calculate_precision(train_output, self._y_train) * 100 + + # Testing + test_output = self.forward(X_test) + test_loss = self.loss.calculate(test_output, y_test) + test_acc = self._calculate_accuracy(test_output, y_test) * 100 + test_prec = self._calculate_precision(test_output, y_test) * 100 + + evaluation_summary = [ + ["Training Loss", train_loss], + ["Training Accuracy", train_acc], + ["Training Precision", train_prec], + ["Test Loss", test_loss], + ["Test Loss", test_acc], + ["Test Loss", test_prec], + ] + + table = tabulate( + evaluation_summary, + tablefmt="double_grid", + numalign="right", + stralign="center", + colalign=("center", "center") + ) + + print(table) + + return { + "train_loss": train_loss, + "train_acc": train_acc, + "train_precision": train_prec, + "test_loss": test_loss, + "test_acc": test_acc, + "test_precision": test_prec, + } + + def _calculate_accuracy(self, output, y_true): + """ + Calculate accuracy on the loss function. + + Parameters: + output (ndarray): The predicted output from the model. + y_true (ndarray): The true labels. + + Returns: + float: The accuracy of the model (between 0 and 1). + """ + accuracy = None + + if self.loss.name == "BinaryCrossEntropy": + predictions = (output > self.loss.threshold).astype(int) + accuracy = np.mean(predictions == y_true) + + elif self.loss.name == "CategoricalCrossEntropy": + predictions = np.argmax(output, axis=1) + true_classes = np.argmax(y_true, axis=1) + accuracy = np.mean(predictions == true_classes) + + return accuracy + + def _calculate_precision(self, output, y_true): + """ + Calculate precision on the loss function. + + Parameters: + output (ndarray): The predicted output from the model (probabilities or logits). + y_true (ndarray): The true labels (one-hot encoded for multiclass or binary labels). + + Returns: + float: The precision of the model (between 0 and 1). + """ + + precision = None + + if self.loss.name == "BinaryCrossEntropy": + predictions = (output > self.loss.threshold).astype(int) + + tp = np.sum((predictions == 1) & (y_true == 1)) + fp = np.sum((predictions == 1) & (y_true == 0)) + + if tp + fp > 0: + precision = tp / (tp + fp) + else: + precision = 0.0 + + elif self.loss.name == "CategoricalCrossEntropy": + predictions = np.argmax(output, axis=1) + true_classes = np.argmax(y_true, axis=1) + + # Precision per class + ppc = [] + nclasses = output.shape[1] + + for class_idx in range(nclasses): + tp = np.sum((predictions == class_idx) & (true_classes == class_idx)) + fp = np.sum((predictions == class_idx) & (true_classes != class_idx)) + + if tp + fp > 0: + ppc.append(tp / (tp + fp)) + else: + ppc.append(0.0) + + precision = np.mean(ppc) + return precision + + \ No newline at end of file From 2f686bf81a5ae9ee3dae9f56f696d0133c3d4826 Mon Sep 17 00:00:00 2001 From: Kibrewossen <47031138+kebtes@users.noreply.github.com> Date: Sat, 19 Apr 2025 22:09:04 +0300 Subject: [PATCH 2/3] Add set_threshold method to BinaryCrossEntropy class for customizable threshold value --- nnf/losses/binary_cross_entropy.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/nnf/losses/binary_cross_entropy.py b/nnf/losses/binary_cross_entropy.py index 97063e4..ae759a6 100644 --- a/nnf/losses/binary_cross_entropy.py +++ b/nnf/losses/binary_cross_entropy.py @@ -47,6 +47,10 @@ def __init__(self): self.output = None self.dinputs = None + # Threshold value used for calculating accuracy in binary classification. + # Defaults to 0.5, but can be customized via the set_threshold() method. + self._threshold = 0.5 + def forward(self, y_pred, y_true): """ Forward pass to compute the binary cross-entropy loss. @@ -79,3 +83,18 @@ def backward(self, y_pred, y_true): y_pred = np.clip(y_pred, 1e-7, 1 - 1e-7) self.dinputs = -(y_true / y_pred - (1 - y_true) / (1 - y_pred)) / samples return self.dinputs + + def set_threshold(self, threshold): + if threshold > 1 or threshold < 0.0: + raise ValueError("threshold value should be in between 0 and 1") + + if threshold == 0: + import warnings + + warnings.warn( + f"Threshold of {threshold} is unusual. Expected range is (0, 1). " + "This may result in incorrect predictions.", + category=UserWarning + ) + + self._threshold = threshold \ No newline at end of file From 4dbd115f16573a997b4e7e0bda66f4f0c84226bd Mon Sep 17 00:00:00 2001 From: Kibrewossen <47031138+kebtes@users.noreply.github.com> Date: Sat, 19 Apr 2025 22:15:27 +0300 Subject: [PATCH 3/3] Add Git LFS disable step in CI workflow --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba568fb..3f87427 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,3 +28,6 @@ jobs: - name: Set Python Path and Run Tests run: | pytest tests/ + + - name: Disable Git LFS in CI + run: git lfs install --skip-smudge