From 7d084be22e7b8758614cd5347bb8b6e92bb5f65c Mon Sep 17 00:00:00 2001
From: Tao Xiong <xt2357@gmail.com>
Date: Tue, 7 Aug 2018 14:36:09 +0800
Subject: [PATCH] bug fix: avoid overflow in softmax Activator

Subtracting max_y from all y makes the output of softmax harder to overflow.
---
 keras_model.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/keras_model.cc b/keras_model.cc
index 6c65550..0e064bb 100644
--- a/keras_model.cc
+++ b/keras_model.cc
@@ -202,6 +202,11 @@ keras::DataChunk* keras::LayerActivation::compute_output(keras::DataChunk* dc) {
         if(y[k] < 0) y[k] = 0;
       }
     } else if(m_activation_type == "softmax") {
+      // avoid overflows
+      auto max_y = *std::max_element(y.begin(), y.end());
+      for (auto &num : y) {
+        num -= max_y;
+      }
       float sum = 0.0;
       for(unsigned int k = 0; k < y.size(); ++k) {
         y[k] = exp(y[k]);