From 9833d5526a7f88646b12c1b425296f65d9c0a719 Mon Sep 17 00:00:00 2001
From: Kibrewossen <47031138+kebtes@users.noreply.github.com>
Date: Sat, 17 May 2025 14:11:45 +0300
Subject: [PATCH 1/2] Add Dropout layer implementation

---
 nnf/layers/__init__.py |  5 ++--
 nnf/layers/dropout.py  | 62 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+), 2 deletions(-)
 create mode 100644 nnf/layers/dropout.py

diff --git a/nnf/layers/__init__.py b/nnf/layers/__init__.py
index 713518c..742bda0 100644
--- a/nnf/layers/__init__.py
+++ b/nnf/layers/__init__.py
@@ -1,4 +1,5 @@
 from nnf.layers.dense import Dense
-from nnf.layers.dense import Layer
+from nnf.layers.base import Layer
+from nnf.layers.dropout import Dropout
 
-__all__ = ['Dense', 'Layer']
\ No newline at end of file
+__all__ = ['Dense', 'Layer', 'Dropout']
\ No newline at end of file
diff --git a/nnf/layers/dropout.py b/nnf/layers/dropout.py
new file mode 100644
index 0000000..0b3e59a
--- /dev/null
+++ b/nnf/layers/dropout.py
@@ -0,0 +1,62 @@
+import numpy as np
+from nnf.layers import Layer
+
+class Dropout(Layer):
+    """
+    Dropout layer implementing inverted dropout.
+
+    During training, it randomly sets a fraction `p` of input units to zero 
+    at each update and scales the rest by `1 / (1 - p)` to maintain the expected output. 
+    During inference, it passes the input through unchanged.
+
+    Attributes:
+        p (float): Dropout probability. Fraction of inputs to drop.
+        mask (np.ndarray): Binary mask used to drop/keep inputs during training.
+        training (bool): Flag indicating whether the layer is in training mode.
+    """
+
+    def __init__(self, p: float = 0.5):
+        """
+        Initializes the Dropout layer.
+
+        Args:
+            p (float): Probability of dropping a unit. Must be between 0 and 1.
+        """
+        super().__init__()
+        if not (0 <= p < 1):
+            raise ValueError("Dropout probability p must be in the range [0, 1).")
+        self.p = p 
+        self.mask = None
+        self.training = True
+
+    def forward(self, X: np.ndarray) -> np.ndarray:
+        """
+        Applies dropout to the input during training, or passes it unchanged during inference.
+
+        Args:
+            X (np.ndarray): Input array of shape (batch_size, features).
+
+        Returns:
+            np.ndarray: Output after applying dropout (during training) or original input (during inference).
+        """
+        if not self.training:
+            return X  # No dropout during inference
+        
+        self.mask = (np.random.rand(*X.shape) > self.p).astype(np.float32)
+        return (X * self.mask) / (1 - self.p)
+
+    def backward(self, doutput: np.ndarray) -> np.ndarray:
+        """
+        Backward pass through the dropout layer. Only propagates gradients 
+        through the neurons that were active during the forward pass.
+
+        Args:
+            doutput (np.ndarray): Upstream gradient.
+
+        Returns:
+            np.ndarray: Gradient of the loss with respect to the input.
+        """
+        if self.mask is None:
+            raise ValueError("Must call forward() before backward().")
+        
+        return (doutput * self.mask) / (1 - self.p)

From c4557aab81049ff5a4215da44d3d8cb5ed422c83 Mon Sep 17 00:00:00 2001
From: Kibrewossen <47031138+kebtes@users.noreply.github.com>
Date: Sat, 17 May 2025 14:11:54 +0300
Subject: [PATCH 2/2] Add unit tests for Dropout layer functionality

---
 tests/test_layers/test_dropout.py | 63 +++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)
 create mode 100644 tests/test_layers/test_dropout.py

diff --git a/tests/test_layers/test_dropout.py b/tests/test_layers/test_dropout.py
new file mode 100644
index 0000000..2bc5ee8
--- /dev/null
+++ b/tests/test_layers/test_dropout.py
@@ -0,0 +1,63 @@
+import numpy as np
+import pytest
+
+from nnf.layers import Dropout
+
+@pytest.fixture
+def sample_input():
+    return np.ones((4, 4), dtype=np.float32)
+
+@pytest.fixture
+def dropout_layer():
+    return Dropout()
+
+def test_forward_training_shape_and_scale(dropout_layer, sample_input):
+    dropout_layer.training = True
+    out = dropout_layer.forward(sample_input)
+
+    assert out.shape == sample_input.shape
+
+    valid_vals = [0.0, 1.0 / (1 - dropout_layer.p)]
+    unique_vals = np.unique(out)
+    for val in unique_vals:
+        assert val in valid_vals
+
+def test_forward_inference_no_change(dropout_layer, sample_input):
+    dropout_layer.training = False
+    out = dropout_layer.forward(sample_input)
+
+    np.testing.assert_array_equal(out, sample_input)
+
+def test_mask_is_binary(dropout_layer, sample_input):
+    dropout_layer.training = True
+    _ = dropout_layer.forward(sample_input)
+    mask = dropout_layer.mask
+
+    assert mask is not None
+    assert mask.shape == sample_input.shape
+
+    unique_vals = np.unique(mask)
+    for val in unique_vals:
+        assert val in [0.0, 1.0]
+
+def test_backward_gradient_masking(dropout_layer, sample_input):
+    dropout_layer.training = True
+    _ = dropout_layer.forward(sample_input)
+
+    upstream_grad = np.full_like(sample_input, 2.0)
+    dx = dropout_layer.backward(upstream_grad)
+
+    expected = (upstream_grad * dropout_layer.mask) / (1 - dropout_layer.p)
+    np.testing.assert_array_almost_equal(dx, expected)
+
+def test_backward_raises_without_forward():
+    dropout = Dropout(p=0.5)
+    upstream_grad = np.ones((4, 4), dtype=np.float32)
+
+    with pytest.raises(ValueError):
+        dropout.backward(upstream_grad)
+
+@pytest.mark.parametrize("invalid_p", [-0.1, 1.0, 1.5])
+def test_invalid_p_raises(invalid_p):
+    with pytest.raises(ValueError):
+        Dropout(p=invalid_p)
\ No newline at end of file