Merge pull request #29 from acompany-develop/feature/otsuka/add_progress_log

come2ry · web-flow · commit f6ff1b51dce8 · 2022-09-12T01:42:32.000+09:00
Add simple progress log
diff --git a/src/ComputationContainer/Client/ComputationToDbGate/ValueTable.cpp b/src/ComputationContainer/Client/ComputationToDbGate/ValueTable.cpp
@@ -1,5 +1,6 @@
 #include "ValueTable.hpp"
 
+#include <chrono>
 #include <unordered_map>
 #include <unordered_set>
 
@@ -76,7 +77,11 @@ std::vector<std::pair<int, int>> intersectionSortedValueIndex(
     // v1, v2がソートされている必要あり
     std::vector<std::pair<int, int>> it_list;
     it_list.reserve(sorted_v1.size());
+
     size_t i1 = 0, i2 = 0;
+    std::uint32_t iterated = 0;
+    spdlog::info("[progress] hjoin: core (0/1)");
+    auto time_from = std::chrono::system_clock::now();
     while (i1 < sorted_v1.size() && i2 < sorted_v2.size())
     {
         if (sorted_v1[i1] == sorted_v2[i2])
@@ -93,7 +98,25 @@ std::vector<std::pair<int, int>> intersectionSortedValueIndex(
         {
             ++i2;
         }
+        if (++iterated % 10 == 0)
+        {
+            auto time_to = std::chrono::system_clock::now();
+            auto dur = std::chrono::duration_cast<std::chrono::milliseconds>(time_to - time_from);
+
+            if (dur.count() >= 5000)
+            {
+                double max_progress =
+                    std::max(i1 * 100.0 / sorted_v1.size(), i2 * 100.0 / sorted_v2.size());
+                spdlog::info("[progress] hjoin: core (0/1): {:>5.2f} %", max_progress);
+
+                time_from = time_to;
+            }
+        }
     }
+    spdlog::info("[progress] hjoin: core (0/1): {:>5.2f} %", 100.0);
+
+    spdlog::info("[progress] hjoin: core (1/1)");
+
     return it_list;
 }
 
diff --git a/src/ComputationContainer/Job/Jobs/LinearRegressionJob.hpp b/src/ComputationContainer/Job/Jobs/LinearRegressionJob.hpp
@@ -62,6 +62,7 @@ class LinearRegressionJob : public JobBase<LinearRegressionJob>
         const std::vector<std::list<int>> &arg
     )
     {
+        spdlog::info("[progress] Linear Regression: learning: pre-processing (0/4)");
         // 入力parse
         auto [x, y] = parse_table(table, arg);
         if (x.empty())
@@ -74,10 +75,18 @@ class LinearRegressionJob : public JobBase<LinearRegressionJob>
         auto mat_y = qmpc::Share::ShareMatrix(y);
 
         // w = (X^T * X)^{-1} * X^T * y
+        spdlog::info("[progress] Linear Regression: learning: mat mul (1/4)");
         auto mat_x_trans = mat_x.transpose();
         auto mat_mul = (mat_x_trans * mat_x);
+
+        spdlog::info("[progress] Linear Regression: learning: mat inverse (2/4)");
         auto mat_mul_inv = mat_mul.inverse();
+
+        spdlog::info("[progress] Linear Regression: learning: mat mul (3/4)");
         auto w = mat_mul_inv * mat_x_trans * mat_y;
+
+        spdlog::info("[progress] Linear Regression: learning: post-procesing (4/4)");
+
         return w.transpose().get_row()[0];
     }
 };
diff --git a/src/ComputationContainer/Model/Models/LinearRegression.hpp b/src/ComputationContainer/Model/Models/LinearRegression.hpp
@@ -1,5 +1,8 @@
 #pragma once
 
+#include <boost/range/adaptor/indexed.hpp>
+#include <chrono>
+
 #include "Client/ComputationToDbGate/Client.hpp"
 #include "ConfigParse/ConfigParse.hpp"
 #include "Model/ModelBase.hpp"
@@ -54,17 +57,42 @@ class LinearRegression : public ModelBase
         const std::string &model_param_job_uuid
     ) const override
     {
+        spdlog::info("[progress] Linear Regression: predict: pre-processing (0/2)");
+
         auto db_client = qmpc::ComputationToDbGate::Client::getInstance();
         auto a_str = db_client->readModelparam(model_param_job_uuid);
         auto a = std::vector<Share>(a_str.begin(), a_str.end());
         auto x = parse_table(table, arg);
 
         std::vector<Share> result;
         result.reserve(a.size());
-        for (const auto &xi : x)
+
+        spdlog::info("[progress] Linear Regression: predict: compute (1/2)");
+        auto time_from = std::chrono::system_clock::now();
+        for (const auto &xi : boost::adaptors::index(x))
         {
-            result.emplace_back(predict_f(a, xi));
+            if (xi.index() % 1000 == 0)
+            {
+                auto time_to = std::chrono::system_clock::now();
+                auto dur =
+                    std::chrono::duration_cast<std::chrono::milliseconds>(time_to - time_from);
+
+                if (dur.count() >= 5000)
+                {
+                    spdlog::info(
+                        "[progress] Linear Regression: predict: compute (1/2): {:>5.2f} %",
+                        xi.index() * 100.0 / x.size()
+                    );
+                    time_from = time_to;
+                }
+            }
+
+            result.emplace_back(predict_f(a, xi.value()));
         }
+        spdlog::info("[progress] Linear Regression: predict: compute (1/2): {:>5.2f} %", 100.0);
+
+        spdlog::info("[progress] Linear Regression: predict: post-processing (2/2)");
+
         return result;
     }
 };
diff --git a/src/ComputationContainer/Model/Models/LogisticRegression.hpp b/src/ComputationContainer/Model/Models/LogisticRegression.hpp
@@ -1,5 +1,8 @@
 #pragma once
 
+#include <boost/range/adaptor/indexed.hpp>
+#include <chrono>
+
 #include "Client/ComputationToDbGate/Client.hpp"
 #include "ConfigParse/ConfigParse.hpp"
 #include "Math/Math.hpp"
@@ -62,17 +65,41 @@ class LogisticRegression : public ModelBase
          * 3. [1] / ([1] + [E]) を計算する
          */
 
+        spdlog::info("[progress] Logistic Regression: predict: pre-processing (0/2)");
         auto db_client = qmpc::ComputationToDbGate::Client::getInstance();
         auto a_str = db_client->readModelparam(model_param_job_uuid);
         auto a = std::vector<Share>(a_str.begin(), a_str.end());
         auto x = parse_table(table, arg);
 
         std::vector<Share> result;
         result.reserve(a.size());
-        for (const auto &xi : x)
+
+        spdlog::info("[progress] Logistic Regression: predict: compute (1/2)");
+        auto time_from = std::chrono::system_clock::now();
+        for (const auto &xi : boost::adaptors::index(x))
         {
-            result.emplace_back(predict_f(a, xi));
+            if (xi.index() % 1000 == 0)
+            {
+                auto time_to = std::chrono::system_clock::now();
+                auto dur =
+                    std::chrono::duration_cast<std::chrono::milliseconds>(time_to - time_from);
+
+                if (dur.count() >= 5000)
+                {
+                    spdlog::info(
+                        "[progress] Logistic Regression: predict: compute (1/2): {:>5.2f} %",
+                        xi.index() * 100.0 / x.size()
+                    );
+                    time_from = time_to;
+                }
+            }
+
+            result.emplace_back(predict_f(a, xi.value()));
         }
+        spdlog::info("[progress] Logistic Regression: predict: compute (1/): {:>5.2f} %", 100.0);
+
+        spdlog::info("[progress] Logistic Regression: predict: post-processing (2/2)");
+
         return result;
     }
 };
diff --git a/src/ComputationContainer/Optimizer/SGD.hpp b/src/ComputationContainer/Optimizer/SGD.hpp
@@ -29,6 +29,7 @@ class SGD : public qmpc::Optimizer::OptInterface
         int iterationNum, const interface &f, const std::vector<Share> &theta
     ) const override
     {
+        spdlog::info("[progress] SGD optimize: remain: {}", iterationNum);
         if (iterationNum == 0)
         {
             return theta;

Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,7 @@ class SGD : public qmpc::Optimizer::OptInterface`
`29`	`29`	`int iterationNum, const interface &f, const std::vector<Share> &theta`
`30`	`30`	`) const override`
`31`	`31`	`{`
	`32`	`+ spdlog::info("[progress] SGD optimize: remain: {}", iterationNum);`
`32`	`33`	`if (iterationNum == 0)`
`33`	`34`	`{`
`34`	`35`	`return theta;`