From b2d6f452249b4a5dbb415305daabc62d60402dcf Mon Sep 17 00:00:00 2001 From: Steven Atkinson Date: Mon, 26 Jan 2026 18:03:17 -0800 Subject: [PATCH 1/5] Add Softsign activation function and corresponding tests - Introduced the Softsign activation function to the activations module. - Updated ActivationType enum to include Softsign. - Enhanced ActivationConfig to support Softsign with an alpha parameter. - Implemented the ActivationSoftsign class for applying the Softsign function. - Added unit tests for Softsign, covering core functionality, initialization, and JSON configuration parsing. - Updated run_tests.cpp to include tests for Softsign functionality. --- NAM/activations.cpp | 17 +++++- NAM/activations.h | 26 ++++++++- tools/run_tests.cpp | 7 +++ tools/test/test_activations.cpp | 100 +++++++++++++++++++++++++++++++- 4 files changed, 146 insertions(+), 4 deletions(-) diff --git a/NAM/activations.cpp b/NAM/activations.cpp index 4127d5a..4e7f0ec 100644 --- a/NAM/activations.cpp +++ b/NAM/activations.cpp @@ -11,6 +11,7 @@ static nam::activations::ActivationSigmoid _SIGMOID; static nam::activations::ActivationSwish _SWISH; static nam::activations::ActivationHardSwish _HARD_SWISH; static nam::activations::ActivationLeakyHardTanh _LEAKY_HARD_TANH; +static nam::activations::ActivationSoftsign _SOFTSIGN; bool nam::activations::Activation::using_fast_tanh = false; @@ -31,7 +32,8 @@ std::unordered_map nam::activati {"SiLU", make_singleton_ptr(_SWISH)}, {"Hardswish", make_singleton_ptr(_HARD_SWISH)}, {"LeakyHardtanh", make_singleton_ptr(_LEAKY_HARD_TANH)}, - {"PReLU", make_singleton_ptr(_PRELU)}}; + {"PReLU", make_singleton_ptr(_PRELU)}, + {"Softsign", make_singleton_ptr(_SOFTSIGN)}}; nam::activations::Activation::Ptr tanh_bak = nullptr; nam::activations::Activation::Ptr sigmoid_bak = nullptr; @@ -68,7 +70,8 @@ nam::activations::ActivationConfig nam::activations::ActivationConfig::from_json {"SiLU", ActivationType::SiLU}, {"Hardswish", ActivationType::Hardswish}, {"LeakyHardtanh", ActivationType::LeakyHardtanh}, - {"LeakyHardTanh", ActivationType::LeakyHardtanh} // Support both casings + {"LeakyHardTanh", ActivationType::LeakyHardtanh}, // Support both casings + {"Softsign", ActivationType::Softsign} }; // If it's a string, simple lookup @@ -118,6 +121,10 @@ nam::activations::ActivationConfig nam::activations::ActivationConfig::from_json config.min_slope = j.value("min_slope", 0.01f); config.max_slope = j.value("max_slope", 0.01f); } + else if (config.type == ActivationType::Softsign) + { + config.alpha = j.value("alpha", 1.0f); + } return config; } @@ -156,6 +163,12 @@ nam::activations::Activation::Ptr nam::activations::Activation::get_activation(c return std::make_shared(config.min_val.value_or(-1.0f), config.max_val.value_or(1.0f), config.min_slope.value_or(0.01f), config.max_slope.value_or(0.01f)); + case ActivationType::Softsign: + if (config.alpha.has_value()) + { + return std::make_shared(config.alpha.value()); + } + return _activations["Softsign"]; default: return nullptr; } } diff --git a/NAM/activations.h b/NAM/activations.h index d30e4dd..0563806 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -33,7 +33,8 @@ enum class ActivationType Sigmoid, SiLU, // aka Swish Hardswish, - LeakyHardtanh + LeakyHardtanh, + Softsign }; // Strongly-typed activation configuration @@ -48,6 +49,7 @@ struct ActivationConfig std::optional max_val; // LeakyHardtanh std::optional min_slope; // LeakyHardtanh std::optional max_slope; // LeakyHardtanh + std::optional alpha; // Softsign // Convenience constructors static ActivationConfig simple(ActivationType t); @@ -130,6 +132,11 @@ inline float hardswish(float x) } } +inline float softsign(float x, float alpha = 1.0f) +{ + return x / (alpha + fabsf(x)); +} + class Activation { public: @@ -333,6 +340,23 @@ class ActivationHardSwish : public Activation } }; +class ActivationSoftsign : public Activation +{ +public: + ActivationSoftsign() = default; + ActivationSoftsign(float a) { alpha = a; } + void apply(float* data, long size) override + { + for (long pos = 0; pos < size; pos++) + { + data[pos] = softsign(data[pos], alpha); + } + } + +private: + float alpha = 1.0f; +}; + class FastLUTActivation : public Activation { public: diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index 218881e..56abfec 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -38,6 +38,10 @@ int main() test_activations::TestLeakyReLU::test_get_by_init(); test_activations::TestLeakyReLU::test_get_by_str(); + test_activations::TestSoftsign::test_core_function(); + test_activations::TestSoftsign::test_get_by_init(); + test_activations::TestSoftsign::test_get_by_str(); + test_lut::TestFastLUT::test_sigmoid(); test_lut::TestFastLUT::test_tanh(); @@ -53,9 +57,12 @@ int main() test_activations::TestTypedActivationConfig::test_prelu_single_slope_config(); test_activations::TestTypedActivationConfig::test_prelu_multi_slope_config(); test_activations::TestTypedActivationConfig::test_leaky_hardtanh_config(); + test_activations::TestTypedActivationConfig::test_softsign_config(); test_activations::TestTypedActivationConfig::test_from_json_string(); test_activations::TestTypedActivationConfig::test_from_json_object(); test_activations::TestTypedActivationConfig::test_from_json_prelu_multi(); + test_activations::TestTypedActivationConfig::test_from_json_softsign_string(); + test_activations::TestTypedActivationConfig::test_from_json_softsign_object(); test_activations::TestTypedActivationConfig::test_unknown_activation_throws(); test_dsp::test_construct(); diff --git a/tools/test/test_activations.cpp b/tools/test/test_activations.cpp index abbdd23..6f80bf5 100644 --- a/tools/test/test_activations.cpp +++ b/tools/test/test_activations.cpp @@ -120,6 +120,66 @@ class TestLeakyReLU } }; }; +class TestSoftsign +{ +public: + static void test_core_function() + { + auto TestCase = [](float input, float alpha, float expectedOutput) { + float actualOutput = nam::activations::softsign(input, alpha); + assert(fabs(actualOutput - expectedOutput) < 1e-6); + }; + // Test cases for softsign: x / (alpha + |x|) + // With alpha = 1.0 (default): + TestCase(0.0f, 1.0f, 0.0f); // 0 / (1 + 0) = 0 + TestCase(1.0f, 1.0f, 0.5f); // 1 / (1 + 1) = 0.5 + TestCase(-1.0f, 1.0f, -0.5f); // -1 / (1 + 1) = -0.5 + TestCase(2.0f, 1.0f, 2.0f / 3.0f); // 2 / (1 + 2) = 2/3 + TestCase(-2.0f, 1.0f, -2.0f / 3.0f); // -2 / (1 + 2) = -2/3 + + // With alpha = 0.5: + TestCase(1.0f, 0.5f, 1.0f / 1.5f); // 1 / (0.5 + 1) = 1/1.5 + TestCase(-1.0f, 0.5f, -1.0f / 1.5f); // -1 / (0.5 + 1) = -1/1.5 + }; + + static void test_get_by_init() + { + auto a = nam::activations::ActivationSoftsign(1.0f); + _test_class(&a); + } + + // Get the singleton and test it + static void test_get_by_str() + { + const std::string name = "Softsign"; + auto a = nam::activations::Activation::get_activation(name); + _test_class(a.get()); + } + +private: + // Put the class through its paces + static void _test_class(nam::activations::Activation* a) + { + std::vector inputs, expectedOutputs; + + inputs.push_back(0.0f); + expectedOutputs.push_back(0.0f); + + inputs.push_back(1.0f); + expectedOutputs.push_back(0.5f); // 1 / (1 + 1) = 0.5 + + inputs.push_back(-1.0f); + expectedOutputs.push_back(-0.5f); // -1 / (1 + 1) = -0.5 + + a->apply(inputs.data(), (long)inputs.size()); + for (auto itActual = inputs.begin(), itExpected = expectedOutputs.begin(); itActual != inputs.end(); + ++itActual, ++itExpected) + { + assert(fabs(*itActual - *itExpected) < 1e-6); + } + }; +}; + class TestPReLU { public: @@ -217,7 +277,7 @@ class TestTypedActivationConfig nam::activations::ActivationType::Tanh, nam::activations::ActivationType::Hardtanh, nam::activations::ActivationType::Fasttanh, nam::activations::ActivationType::ReLU, nam::activations::ActivationType::Sigmoid, nam::activations::ActivationType::SiLU, - nam::activations::ActivationType::Hardswish}; + nam::activations::ActivationType::Hardswish, nam::activations::ActivationType::Softsign}; for (auto type : types) { @@ -296,6 +356,24 @@ class TestTypedActivationConfig assert(act != nullptr); } + static void test_softsign_config() + { + // Test Softsign with custom alpha + nam::activations::ActivationConfig config; + config.type = nam::activations::ActivationType::Softsign; + config.alpha = 0.5f; + + auto act = nam::activations::Activation::get_activation(config); + assert(act != nullptr); + + // Verify the behavior + std::vector data = {-1.0f, 0.0f, 1.0f}; + act->apply(data.data(), (long)data.size()); + assert(fabs(data[0] - (-1.0f / 1.5f)) < 1e-6); // -1 / (0.5 + 1) = -1/1.5 + assert(fabs(data[1] - 0.0f) < 1e-6); + assert(fabs(data[2] - (1.0f / 1.5f)) < 1e-6); // 1 / (0.5 + 1) = 1/1.5 + } + static void test_from_json_string() { // Test from_json with string input @@ -324,6 +402,26 @@ class TestTypedActivationConfig assert(config.negative_slopes.value().size() == 4); } + static void test_from_json_softsign_string() + { + // Test from_json with Softsign as string (default alpha) + nlohmann::json j = "Softsign"; + auto config = nam::activations::ActivationConfig::from_json(j); + assert(config.type == nam::activations::ActivationType::Softsign); + // When parsing from string, alpha is not set, but get_activation will use default + assert(!config.alpha.has_value()); + } + + static void test_from_json_softsign_object() + { + // Test from_json with Softsign as object with custom alpha + nlohmann::json j = {{"type", "Softsign"}, {"alpha", 0.5f}}; + auto config = nam::activations::ActivationConfig::from_json(j); + assert(config.type == nam::activations::ActivationType::Softsign); + assert(config.alpha.has_value()); + assert(fabs(config.alpha.value() - 0.5f) < 1e-6); + } + static void test_unknown_activation_throws() { // Test that unknown activation type throws From 6d380bc63d4013580dafa87dc09a5e5053a0318f Mon Sep 17 00:00:00 2001 From: Steven Atkinson Date: Mon, 26 Jan 2026 18:03:35 -0800 Subject: [PATCH 2/5] Formatting --- NAM/activations.cpp | 3 +-- tools/test/test_activations.cpp | 14 +++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/NAM/activations.cpp b/NAM/activations.cpp index 4e7f0ec..e6b3748 100644 --- a/NAM/activations.cpp +++ b/NAM/activations.cpp @@ -71,8 +71,7 @@ nam::activations::ActivationConfig nam::activations::ActivationConfig::from_json {"Hardswish", ActivationType::Hardswish}, {"LeakyHardtanh", ActivationType::LeakyHardtanh}, {"LeakyHardTanh", ActivationType::LeakyHardtanh}, // Support both casings - {"Softsign", ActivationType::Softsign} - }; + {"Softsign", ActivationType::Softsign}}; // If it's a string, simple lookup if (j.is_string()) diff --git a/tools/test/test_activations.cpp b/tools/test/test_activations.cpp index 6f80bf5..83dc422 100644 --- a/tools/test/test_activations.cpp +++ b/tools/test/test_activations.cpp @@ -131,12 +131,12 @@ class TestSoftsign }; // Test cases for softsign: x / (alpha + |x|) // With alpha = 1.0 (default): - TestCase(0.0f, 1.0f, 0.0f); // 0 / (1 + 0) = 0 - TestCase(1.0f, 1.0f, 0.5f); // 1 / (1 + 1) = 0.5 - TestCase(-1.0f, 1.0f, -0.5f); // -1 / (1 + 1) = -0.5 + TestCase(0.0f, 1.0f, 0.0f); // 0 / (1 + 0) = 0 + TestCase(1.0f, 1.0f, 0.5f); // 1 / (1 + 1) = 0.5 + TestCase(-1.0f, 1.0f, -0.5f); // -1 / (1 + 1) = -0.5 TestCase(2.0f, 1.0f, 2.0f / 3.0f); // 2 / (1 + 2) = 2/3 TestCase(-2.0f, 1.0f, -2.0f / 3.0f); // -2 / (1 + 2) = -2/3 - + // With alpha = 0.5: TestCase(1.0f, 0.5f, 1.0f / 1.5f); // 1 / (0.5 + 1) = 1/1.5 TestCase(-1.0f, 0.5f, -1.0f / 1.5f); // -1 / (0.5 + 1) = -1/1.5 @@ -274,9 +274,9 @@ class TestTypedActivationConfig { // Test that all simple activation types work std::vector types = { - nam::activations::ActivationType::Tanh, nam::activations::ActivationType::Hardtanh, - nam::activations::ActivationType::Fasttanh, nam::activations::ActivationType::ReLU, - nam::activations::ActivationType::Sigmoid, nam::activations::ActivationType::SiLU, + nam::activations::ActivationType::Tanh, nam::activations::ActivationType::Hardtanh, + nam::activations::ActivationType::Fasttanh, nam::activations::ActivationType::ReLU, + nam::activations::ActivationType::Sigmoid, nam::activations::ActivationType::SiLU, nam::activations::ActivationType::Hardswish, nam::activations::ActivationType::Softsign}; for (auto type : types) From c85c33d48bd6a29c41c8c7e73282d04deabd76d9 Mon Sep 17 00:00:00 2001 From: Steven Atkinson Date: Mon, 26 Jan 2026 18:05:03 -0800 Subject: [PATCH 3/5] use Softsign in A2-max --- example_models/wavenet_a2_max.nam | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example_models/wavenet_a2_max.nam b/example_models/wavenet_a2_max.nam index 1a48c95..aa0f933 100644 --- a/example_models/wavenet_a2_max.nam +++ b/example_models/wavenet_a2_max.nam @@ -1239,8 +1239,8 @@ 2 ], "activation": { - "type": "PReLU", - "negative_slope": 0.015 + "type": "Softsign", + "alpha": 4.0 }, "gating_mode": "none", "head_bias": true, From 4d5e28188ce064a3ecf1ed4caf106c043c74d8d2 Mon Sep 17 00:00:00 2001 From: Steven Atkinson Date: Mon, 26 Jan 2026 18:17:26 -0800 Subject: [PATCH 4/5] Refactor Softsign activation implementation and tests - Simplified the Softsign function by removing the alpha parameter, defaulting to 1.0. - Updated related tests to reflect the new Softsign behavior without alpha. - Adjusted ActivationConfig to remove the alpha field for Softsign. - Enhanced test cases to ensure correct functionality of the Softsign activation. - Modified JSON parsing for Softsign to ignore alpha parameter when provided. --- NAM/activations.cpp | 11 +--------- NAM/activations.h | 12 +++-------- tools/test/test_activations.cpp | 38 ++++++++++++--------------------- 3 files changed, 18 insertions(+), 43 deletions(-) diff --git a/NAM/activations.cpp b/NAM/activations.cpp index e6b3748..a476520 100644 --- a/NAM/activations.cpp +++ b/NAM/activations.cpp @@ -120,10 +120,6 @@ nam::activations::ActivationConfig nam::activations::ActivationConfig::from_json config.min_slope = j.value("min_slope", 0.01f); config.max_slope = j.value("max_slope", 0.01f); } - else if (config.type == ActivationType::Softsign) - { - config.alpha = j.value("alpha", 1.0f); - } return config; } @@ -162,12 +158,7 @@ nam::activations::Activation::Ptr nam::activations::Activation::get_activation(c return std::make_shared(config.min_val.value_or(-1.0f), config.max_val.value_or(1.0f), config.min_slope.value_or(0.01f), config.max_slope.value_or(0.01f)); - case ActivationType::Softsign: - if (config.alpha.has_value()) - { - return std::make_shared(config.alpha.value()); - } - return _activations["Softsign"]; + case ActivationType::Softsign: return _activations["Softsign"]; default: return nullptr; } } diff --git a/NAM/activations.h b/NAM/activations.h index 0563806..68d5025 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -49,7 +49,6 @@ struct ActivationConfig std::optional max_val; // LeakyHardtanh std::optional min_slope; // LeakyHardtanh std::optional max_slope; // LeakyHardtanh - std::optional alpha; // Softsign // Convenience constructors static ActivationConfig simple(ActivationType t); @@ -132,9 +131,9 @@ inline float hardswish(float x) } } -inline float softsign(float x, float alpha = 1.0f) +inline float softsign(float x) { - return x / (alpha + fabsf(x)); + return x / (1.0f + fabsf(x)); } class Activation @@ -343,18 +342,13 @@ class ActivationHardSwish : public Activation class ActivationSoftsign : public Activation { public: - ActivationSoftsign() = default; - ActivationSoftsign(float a) { alpha = a; } void apply(float* data, long size) override { for (long pos = 0; pos < size; pos++) { - data[pos] = softsign(data[pos], alpha); + data[pos] = softsign(data[pos]); } } - -private: - float alpha = 1.0f; }; class FastLUTActivation : public Activation diff --git a/tools/test/test_activations.cpp b/tools/test/test_activations.cpp index 83dc422..a8dd705 100644 --- a/tools/test/test_activations.cpp +++ b/tools/test/test_activations.cpp @@ -125,26 +125,21 @@ class TestSoftsign public: static void test_core_function() { - auto TestCase = [](float input, float alpha, float expectedOutput) { - float actualOutput = nam::activations::softsign(input, alpha); + auto TestCase = [](float input, float expectedOutput) { + float actualOutput = nam::activations::softsign(input); assert(fabs(actualOutput - expectedOutput) < 1e-6); }; - // Test cases for softsign: x / (alpha + |x|) - // With alpha = 1.0 (default): - TestCase(0.0f, 1.0f, 0.0f); // 0 / (1 + 0) = 0 - TestCase(1.0f, 1.0f, 0.5f); // 1 / (1 + 1) = 0.5 - TestCase(-1.0f, 1.0f, -0.5f); // -1 / (1 + 1) = -0.5 - TestCase(2.0f, 1.0f, 2.0f / 3.0f); // 2 / (1 + 2) = 2/3 - TestCase(-2.0f, 1.0f, -2.0f / 3.0f); // -2 / (1 + 2) = -2/3 - - // With alpha = 0.5: - TestCase(1.0f, 0.5f, 1.0f / 1.5f); // 1 / (0.5 + 1) = 1/1.5 - TestCase(-1.0f, 0.5f, -1.0f / 1.5f); // -1 / (0.5 + 1) = -1/1.5 + // Test cases for softsign: x / (1 + |x|) + TestCase(0.0f, 0.0f); // 0 / (1 + 0) = 0 + TestCase(1.0f, 0.5f); // 1 / (1 + 1) = 0.5 + TestCase(-1.0f, -0.5f); // -1 / (1 + 1) = -0.5 + TestCase(2.0f, 2.0f / 3.0f); // 2 / (1 + 2) = 2/3 + TestCase(-2.0f, -2.0f / 3.0f); // -2 / (1 + 2) = -2/3 }; static void test_get_by_init() { - auto a = nam::activations::ActivationSoftsign(1.0f); + auto a = nam::activations::ActivationSoftsign(); _test_class(&a); } @@ -358,10 +353,9 @@ class TestTypedActivationConfig static void test_softsign_config() { - // Test Softsign with custom alpha + // Test Softsign configuration nam::activations::ActivationConfig config; config.type = nam::activations::ActivationType::Softsign; - config.alpha = 0.5f; auto act = nam::activations::Activation::get_activation(config); assert(act != nullptr); @@ -369,9 +363,9 @@ class TestTypedActivationConfig // Verify the behavior std::vector data = {-1.0f, 0.0f, 1.0f}; act->apply(data.data(), (long)data.size()); - assert(fabs(data[0] - (-1.0f / 1.5f)) < 1e-6); // -1 / (0.5 + 1) = -1/1.5 + assert(fabs(data[0] - (-0.5f)) < 1e-6); // -1 / (1 + 1) = -0.5 assert(fabs(data[1] - 0.0f) < 1e-6); - assert(fabs(data[2] - (1.0f / 1.5f)) < 1e-6); // 1 / (0.5 + 1) = 1/1.5 + assert(fabs(data[2] - 0.5f) < 1e-6); // 1 / (1 + 1) = 0.5 } static void test_from_json_string() @@ -404,22 +398,18 @@ class TestTypedActivationConfig static void test_from_json_softsign_string() { - // Test from_json with Softsign as string (default alpha) + // Test from_json with Softsign as string nlohmann::json j = "Softsign"; auto config = nam::activations::ActivationConfig::from_json(j); assert(config.type == nam::activations::ActivationType::Softsign); - // When parsing from string, alpha is not set, but get_activation will use default - assert(!config.alpha.has_value()); } static void test_from_json_softsign_object() { - // Test from_json with Softsign as object with custom alpha + // Test from_json with Softsign as object (alpha parameter is ignored) nlohmann::json j = {{"type", "Softsign"}, {"alpha", 0.5f}}; auto config = nam::activations::ActivationConfig::from_json(j); assert(config.type == nam::activations::ActivationType::Softsign); - assert(config.alpha.has_value()); - assert(fabs(config.alpha.value() - 0.5f) < 1e-6); } static void test_unknown_activation_throws() From c05dc77104c76588824f0d2303130b1ff44123aa Mon Sep 17 00:00:00 2001 From: Steven Atkinson Date: Mon, 26 Jan 2026 18:18:50 -0800 Subject: [PATCH 5/5] Remove alpha from A2-max --- example_models/wavenet_a2_max.nam | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/example_models/wavenet_a2_max.nam b/example_models/wavenet_a2_max.nam index aa0f933..f54f396 100644 --- a/example_models/wavenet_a2_max.nam +++ b/example_models/wavenet_a2_max.nam @@ -1239,8 +1239,7 @@ 2 ], "activation": { - "type": "Softsign", - "alpha": 4.0 + "type": "Softsign" }, "gating_mode": "none", "head_bias": true,