From 16b11d7aa45e8de85d228cc94e38304945c0f38d Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Wed, 24 May 2017 16:07:30 -0700 Subject: [PATCH 01/36] Infra for default explanations --- src/libclipper/include/clipper/datatypes.hpp | 3 ++- src/libclipper/include/clipper/exceptions.hpp | 9 +++++++++ src/libclipper/src/datatypes.cpp | 5 +++-- src/libclipper/src/exceptions.cpp | 8 ++++++++ src/libclipper/src/query_processor.cpp | 13 +++++++++---- src/libclipper/src/selection_policies.cpp | 2 ++ 6 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/libclipper/include/clipper/datatypes.hpp b/src/libclipper/include/clipper/datatypes.hpp index 46e91ffc2..f7cc0a178 100644 --- a/src/libclipper/include/clipper/datatypes.hpp +++ b/src/libclipper/include/clipper/datatypes.hpp @@ -230,7 +230,7 @@ class Response { ~Response() = default; Response(Query query, QueryId query_id, const long duration_micros, - Output output, const bool is_default); + Output output, const bool is_default, const std::string default_explanation); // default copy constructors Response(const Response &) = default; @@ -247,6 +247,7 @@ class Response { long duration_micros_; Output output_; bool output_is_default_; + std::string default_explanation_; }; class Feedback { diff --git a/src/libclipper/include/clipper/exceptions.hpp b/src/libclipper/include/clipper/exceptions.hpp index c9d9c4304..3c2d0febf 100644 --- a/src/libclipper/include/clipper/exceptions.hpp +++ b/src/libclipper/include/clipper/exceptions.hpp @@ -15,6 +15,15 @@ class PredictError : public std::runtime_error { const std::string msg_; }; +class NoModelsFoundError : public std::runtime_error { + public: + NoModelsFoundError(); + const char* what() const noexcept; + + private: + std::string msg_; +}; + } // namespace clipper #endif // CLIPPER_EXCEPTIONS_HPP diff --git a/src/libclipper/src/datatypes.cpp b/src/libclipper/src/datatypes.cpp index dc4fc4fac..a15d71d97 100644 --- a/src/libclipper/src/datatypes.cpp +++ b/src/libclipper/src/datatypes.cpp @@ -305,12 +305,13 @@ Query::Query(std::string label, long user_id, std::shared_ptr input, create_time_(std::chrono::high_resolution_clock::now()) {} Response::Response(Query query, QueryId query_id, const long duration_micros, - Output output, const bool output_is_default) + Output output, const bool output_is_default, const std::string default_explanation) : query_(std::move(query)), query_id_(query_id), duration_micros_(duration_micros), output_(std::move(output)), - output_is_default_(output_is_default) {} + output_is_default_(output_is_default), + default_explanation_(default_explanation){} std::string Response::debug_string() const noexcept { std::string debug; diff --git a/src/libclipper/src/exceptions.cpp b/src/libclipper/src/exceptions.cpp index ce1e3dcae..70236fb3a 100644 --- a/src/libclipper/src/exceptions.cpp +++ b/src/libclipper/src/exceptions.cpp @@ -9,4 +9,12 @@ PredictError::PredictError(const std::string msg) : std::runtime_error(msg), msg_(msg) {} const char *PredictError::what() const noexcept { return msg_.c_str(); } + +NoModelsFoundError::NoModelsFoundError() : + std::runtime_error("No models found for query") { +} + +const char* NoModelsFoundError::what() const noexcept { + return "No models found for query"; +} } diff --git a/src/libclipper/src/query_processor.cpp b/src/libclipper/src/query_processor.cpp index d689530ae..ba1d2c103 100644 --- a/src/libclipper/src/query_processor.cpp +++ b/src/libclipper/src/query_processor.cpp @@ -63,8 +63,13 @@ boost::future QueryProcessor::predict(Query query) { std::shared_ptr selection_state = current_policy->deserialize(*state_opt); - std::vector tasks = - current_policy->select_predict_tasks(selection_state, query, query_id); + boost::optional default_explanation; + std::vector tasks; + try { + tasks = current_policy->select_predict_tasks(selection_state, query, query_id); + } catch(const NoModelsFoundError& e) { + default_explanation = "No models found for query"; + } log_info_formatted(LOGGING_TAG_QUERY_PROCESSOR, "Found {} tasks", tasks.size()); @@ -94,12 +99,12 @@ boost::future QueryProcessor::predict(Query query) { boost::promise response_promise; auto response_future = response_promise.get_future(); - // NOTE: We capture the num_completed and completed_flag variables + // NOTE: We capture the num_completed, completed_flag, and default_explanation variables // so that they outlive the composed_futures. response_ready_future.then([ this, query, query_id, response_promise = std::move(response_promise), selection_state, current_policy, task_futures = std::move(task_futures), - num_completed, completed_flag + num_completed, completed_flag, default_explanation ](auto) mutable { vector outputs; diff --git a/src/libclipper/src/selection_policies.cpp b/src/libclipper/src/selection_policies.cpp index 6f8e48c9e..74b405e80 100644 --- a/src/libclipper/src/selection_policies.cpp +++ b/src/libclipper/src/selection_policies.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace clipper { @@ -60,6 +61,7 @@ std::vector DefaultOutputSelectionPolicy::select_predict_tasks( log_error_formatted(LOGGING_TAG_SELECTION_POLICY, "No candidate models for query with label {}", query.label_); + throw NoModelsFoundError(); } else { if (num_candidate_models > 1) { log_error_formatted(LOGGING_TAG_SELECTION_POLICY, From 9e1f49a26de17ce4bb934dc71dabd7e1dcf96a45 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Thu, 25 May 2017 03:12:41 -0600 Subject: [PATCH 02/36] Propagate default explanation to response --- src/frontends/src/query_frontend.hpp | 6 ++++++ src/libclipper/include/clipper/datatypes.hpp | 6 ++++-- src/libclipper/src/datatypes.cpp | 3 ++- src/libclipper/src/query_processor.cpp | 5 +++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/frontends/src/query_frontend.hpp b/src/frontends/src/query_frontend.hpp index e3bb3356e..ef825654d 100644 --- a/src/frontends/src/query_frontend.hpp +++ b/src/frontends/src/query_frontend.hpp @@ -39,6 +39,7 @@ const std::string GET_METRICS = "^/metrics$"; const char* PREDICTION_RESPONSE_KEY_QUERY_ID = "query_id"; const char* PREDICTION_RESPONSE_KEY_OUTPUT = "output"; const char* PREDICTION_RESPONSE_KEY_USED_DEFAULT = "default"; +const char* PREDICTION_RESPONSE_KEY_DEFAULT_EXPLANATION = "default_explanation"; const char* PREDICTION_ERROR_RESPONSE_KEY_ERROR = "error"; const char* PREDICTION_ERROR_RESPONSE_KEY_CAUSE = "cause"; @@ -402,6 +403,11 @@ class RequestHandler { } clipper::json::add_bool(json_response, PREDICTION_RESPONSE_KEY_USED_DEFAULT, query_response.output_is_default_); + if(query_response.output_is_default_ && query_response.default_explanation_) { + clipper::json::add_string(json_response, + PREDICTION_RESPONSE_KEY_DEFAULT_EXPLANATION, + query_response.default_explanation_.get()); + } std::string content = clipper::json::to_json_string(json_response); return content; } diff --git a/src/libclipper/include/clipper/datatypes.hpp b/src/libclipper/include/clipper/datatypes.hpp index f7cc0a178..64fc94f5c 100644 --- a/src/libclipper/include/clipper/datatypes.hpp +++ b/src/libclipper/include/clipper/datatypes.hpp @@ -6,6 +6,8 @@ #include #include +#include + namespace clipper { using ByteBuffer = std::vector; @@ -230,7 +232,7 @@ class Response { ~Response() = default; Response(Query query, QueryId query_id, const long duration_micros, - Output output, const bool is_default, const std::string default_explanation); + Output output, const bool is_default, const boost::optional default_explanation); // default copy constructors Response(const Response &) = default; @@ -247,7 +249,7 @@ class Response { long duration_micros_; Output output_; bool output_is_default_; - std::string default_explanation_; + boost::optional default_explanation_; }; class Feedback { diff --git a/src/libclipper/src/datatypes.cpp b/src/libclipper/src/datatypes.cpp index a15d71d97..27c4ef0b9 100644 --- a/src/libclipper/src/datatypes.cpp +++ b/src/libclipper/src/datatypes.cpp @@ -305,7 +305,8 @@ Query::Query(std::string label, long user_id, std::shared_ptr input, create_time_(std::chrono::high_resolution_clock::now()) {} Response::Response(Query query, QueryId query_id, const long duration_micros, - Output output, const bool output_is_default, const std::string default_explanation) + Output output, const bool output_is_default, + const boost::optional default_explanation) : query_(std::move(query)), query_id_(query_id), duration_micros_(duration_micros), diff --git a/src/libclipper/src/query_processor.cpp b/src/libclipper/src/query_processor.cpp index ba1d2c103..6ebfa9163 100644 --- a/src/libclipper/src/query_processor.cpp +++ b/src/libclipper/src/query_processor.cpp @@ -9,6 +9,7 @@ #define PROVIDES_EXECUTORS #include #include +#include #include #include @@ -68,7 +69,7 @@ boost::future QueryProcessor::predict(Query query) { try { tasks = current_policy->select_predict_tasks(selection_state, query, query_id); } catch(const NoModelsFoundError& e) { - default_explanation = "No models found for query"; + default_explanation = "No registered models found for query"; } log_info_formatted(LOGGING_TAG_QUERY_PROCESSOR, "Found {} tasks", @@ -126,7 +127,7 @@ boost::future QueryProcessor::predict(Query query) { .count(); Response response{query, query_id, duration_micros, final_output.first, - final_output.second}; + final_output.second, default_explanation}; response_promise.set_value(response); }); return response_future; From 73208e3a6d5ae8b34e2923df4e913d757be95d8e Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Sun, 28 May 2017 00:03:19 -0600 Subject: [PATCH 03/36] Format code --- src/frontends/src/query_frontend.hpp | 3 ++- src/libclipper/include/clipper/datatypes.hpp | 3 ++- src/libclipper/include/clipper/exceptions.hpp | 2 +- src/libclipper/src/datatypes.cpp | 2 +- src/libclipper/src/exceptions.cpp | 7 +++---- src/libclipper/src/query_processor.cpp | 18 ++++++++++++------ src/libclipper/src/selection_policies.cpp | 2 +- 7 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/frontends/src/query_frontend.hpp b/src/frontends/src/query_frontend.hpp index ef825654d..d620290c0 100644 --- a/src/frontends/src/query_frontend.hpp +++ b/src/frontends/src/query_frontend.hpp @@ -403,7 +403,8 @@ class RequestHandler { } clipper::json::add_bool(json_response, PREDICTION_RESPONSE_KEY_USED_DEFAULT, query_response.output_is_default_); - if(query_response.output_is_default_ && query_response.default_explanation_) { + if (query_response.output_is_default_ && + query_response.default_explanation_) { clipper::json::add_string(json_response, PREDICTION_RESPONSE_KEY_DEFAULT_EXPLANATION, query_response.default_explanation_.get()); diff --git a/src/libclipper/include/clipper/datatypes.hpp b/src/libclipper/include/clipper/datatypes.hpp index 64fc94f5c..a07448cac 100644 --- a/src/libclipper/include/clipper/datatypes.hpp +++ b/src/libclipper/include/clipper/datatypes.hpp @@ -232,7 +232,8 @@ class Response { ~Response() = default; Response(Query query, QueryId query_id, const long duration_micros, - Output output, const bool is_default, const boost::optional default_explanation); + Output output, const bool is_default, + const boost::optional default_explanation); // default copy constructors Response(const Response &) = default; diff --git a/src/libclipper/include/clipper/exceptions.hpp b/src/libclipper/include/clipper/exceptions.hpp index 3c2d0febf..40baaf61d 100644 --- a/src/libclipper/include/clipper/exceptions.hpp +++ b/src/libclipper/include/clipper/exceptions.hpp @@ -9,7 +9,7 @@ class PredictError : public std::runtime_error { public: PredictError(const std::string msg); - const char *what() const noexcept; + const char* what() const noexcept; private: const std::string msg_; diff --git a/src/libclipper/src/datatypes.cpp b/src/libclipper/src/datatypes.cpp index 27c4ef0b9..34bebb03f 100644 --- a/src/libclipper/src/datatypes.cpp +++ b/src/libclipper/src/datatypes.cpp @@ -312,7 +312,7 @@ Response::Response(Query query, QueryId query_id, const long duration_micros, duration_micros_(duration_micros), output_(std::move(output)), output_is_default_(output_is_default), - default_explanation_(default_explanation){} + default_explanation_(default_explanation) {} std::string Response::debug_string() const noexcept { std::string debug; diff --git a/src/libclipper/src/exceptions.cpp b/src/libclipper/src/exceptions.cpp index 70236fb3a..497f33d87 100644 --- a/src/libclipper/src/exceptions.cpp +++ b/src/libclipper/src/exceptions.cpp @@ -8,11 +8,10 @@ namespace clipper { PredictError::PredictError(const std::string msg) : std::runtime_error(msg), msg_(msg) {} -const char *PredictError::what() const noexcept { return msg_.c_str(); } +const char* PredictError::what() const noexcept { return msg_.c_str(); } -NoModelsFoundError::NoModelsFoundError() : - std::runtime_error("No models found for query") { -} +NoModelsFoundError::NoModelsFoundError() + : std::runtime_error("No models found for query") {} const char* NoModelsFoundError::what() const noexcept { return "No models found for query"; diff --git a/src/libclipper/src/query_processor.cpp b/src/libclipper/src/query_processor.cpp index 6ebfa9163..fa31e7235 100644 --- a/src/libclipper/src/query_processor.cpp +++ b/src/libclipper/src/query_processor.cpp @@ -7,9 +7,9 @@ #include #define PROVIDES_EXECUTORS +#include #include #include -#include #include #include @@ -67,8 +67,9 @@ boost::future QueryProcessor::predict(Query query) { boost::optional default_explanation; std::vector tasks; try { - tasks = current_policy->select_predict_tasks(selection_state, query, query_id); - } catch(const NoModelsFoundError& e) { + tasks = + current_policy->select_predict_tasks(selection_state, query, query_id); + } catch (const NoModelsFoundError& e) { default_explanation = "No registered models found for query"; } @@ -100,7 +101,8 @@ boost::future QueryProcessor::predict(Query query) { boost::promise response_promise; auto response_future = response_promise.get_future(); - // NOTE: We capture the num_completed, completed_flag, and default_explanation variables + // NOTE: We capture the num_completed, completed_flag, and default_explanation + // variables // so that they outlive the composed_futures. response_ready_future.then([ this, query, query_id, response_promise = std::move(response_promise), @@ -126,8 +128,12 @@ boost::future QueryProcessor::predict(Query query) { end - query.create_time_) .count(); - Response response{query, query_id, duration_micros, final_output.first, - final_output.second, default_explanation}; + Response response{query, + query_id, + duration_micros, + final_output.first, + final_output.second, + default_explanation}; response_promise.set_value(response); }); return response_future; diff --git a/src/libclipper/src/selection_policies.cpp b/src/libclipper/src/selection_policies.cpp index 74b405e80..a0b2b2289 100644 --- a/src/libclipper/src/selection_policies.cpp +++ b/src/libclipper/src/selection_policies.cpp @@ -7,11 +7,11 @@ #include #include +#include #include #include #include #include -#include namespace clipper { From 8d0ba82a3767dc734ee453df3a82dd9f9ab9ab22 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Sun, 28 May 2017 01:05:01 -0600 Subject: [PATCH 04/36] Add latency default expl, format --- src/libclipper/src/query_processor.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libclipper/src/query_processor.cpp b/src/libclipper/src/query_processor.cpp index fa31e7235..26817dea1 100644 --- a/src/libclipper/src/query_processor.cpp +++ b/src/libclipper/src/query_processor.cpp @@ -112,11 +112,18 @@ boost::future QueryProcessor::predict(Query query) { vector outputs; vector used_models; + bool all_tasks_timed_out = true; for (auto r = task_futures.begin(); r != task_futures.end(); ++r) { if ((*r).is_ready()) { outputs.push_back((*r).get()); + all_tasks_timed_out = false; } } + if (all_tasks_timed_out && !task_futures.empty() && !default_explanation) { + default_explanation = + "Failed to retrieve a prediction response within the specified " + "latency SLO"; + } std::pair final_output = current_policy->combine_predictions(selection_state, query, outputs); From 0a2bdf8c719ded44e4e731b74de3877f51529bcd Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Sun, 28 May 2017 01:07:07 -0600 Subject: [PATCH 05/36] Remove msg_ from exception --- src/libclipper/include/clipper/exceptions.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libclipper/include/clipper/exceptions.hpp b/src/libclipper/include/clipper/exceptions.hpp index 40baaf61d..831a54dbf 100644 --- a/src/libclipper/include/clipper/exceptions.hpp +++ b/src/libclipper/include/clipper/exceptions.hpp @@ -19,9 +19,6 @@ class NoModelsFoundError : public std::runtime_error { public: NoModelsFoundError(); const char* what() const noexcept; - - private: - std::string msg_; }; } // namespace clipper From b257318e1eba1e5cfaa288d6a8eceec394e24ada Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Sun, 28 May 2017 04:08:17 -0600 Subject: [PATCH 06/36] Fix tests, update json schema --- src/frontends/CMakeLists.txt | 2 +- src/frontends/src/query_frontend.hpp | 1 + src/frontends/src/query_frontend_tests.cpp | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/frontends/CMakeLists.txt b/src/frontends/CMakeLists.txt index d31660fc9..b874199e5 100644 --- a/src/frontends/CMakeLists.txt +++ b/src/frontends/CMakeLists.txt @@ -14,5 +14,5 @@ add_executable(frontendtests EXCLUDE_FROM_ALL src/frontend_tests.cpp src/query_frontend_tests.cpp ) -target_link_libraries(frontendtests clipper httpserver gtest gmock_main rapidjson cxxopts) +target_link_libraries(frontendtests clipper boost httpserver gtest gmock_main rapidjson cxxopts) add_dependencies(unittests frontendtests) diff --git a/src/frontends/src/query_frontend.hpp b/src/frontends/src/query_frontend.hpp index d620290c0..a35b487cc 100644 --- a/src/frontends/src/query_frontend.hpp +++ b/src/frontends/src/query_frontend.hpp @@ -379,6 +379,7 @@ class RequestHandler { * "query_id" := int, * "output" := float, * "default" := boolean + * "default_explanation" := string (optional) * } */ static const std::string get_prediction_response_content( diff --git a/src/frontends/src/query_frontend_tests.cpp b/src/frontends/src/query_frontend_tests.cpp index 330977a63..17e8927d9 100644 --- a/src/frontends/src/query_frontend_tests.cpp +++ b/src/frontends/src/query_frontend_tests.cpp @@ -1,6 +1,8 @@ #include #include +#include + #include #include #include @@ -18,7 +20,7 @@ class MockQueryProcessor { MockQueryProcessor() = default; boost::future predict(Query query) { Response response(query, 3, 5, Output("-1.0", {std::make_pair("m", 1)}), - false); + false, boost::optional{}); return boost::make_ready_future(response); } boost::future update(FeedbackQuery /*feedback*/) { From 6daa18e4f6091c68e60a755920cd9fa1702252fd Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Sun, 28 May 2017 16:01:08 -0600 Subject: [PATCH 07/36] Fix test case --- src/libclipper/test/selection_policies_test.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libclipper/test/selection_policies_test.cpp b/src/libclipper/test/selection_policies_test.cpp index 92f0aede8..44d093685 100644 --- a/src/libclipper/test/selection_policies_test.cpp +++ b/src/libclipper/test/selection_policies_test.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include using namespace clipper; @@ -31,9 +32,9 @@ TEST_F(DefaultOutputSelectionPolicyTest, 1000, DefaultOutputSelectionPolicy::get_name(), {}}; - auto zero_models_tasks = - policy_.select_predict_tasks(nullptr, zero_candidate_models_query, 0); - EXPECT_EQ(zero_models_tasks.size(), (size_t)0); + ASSERT_THROW( + policy_.select_predict_tasks(nullptr, zero_candidate_models_query, 0), + NoModelsFoundError); } TEST_F(DefaultOutputSelectionPolicyTest, From f58fda6b85ebe38b186686bf68afbc3e52604042 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Mon, 29 May 2017 21:29:27 -0600 Subject: [PATCH 08/36] Basic manager tests --- clipper_admin/clipper_manager.py | 2 +- management/test/manager_test.py | 71 ++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 management/test/manager_test.py diff --git a/clipper_admin/clipper_manager.py b/clipper_admin/clipper_manager.py index bdca607a1..83b2556fa 100644 --- a/clipper_admin/clipper_manager.py +++ b/clipper_admin/clipper_manager.py @@ -432,7 +432,7 @@ def get_all_models(self, verbose=False): Parameters ---------- verbose : bool - If set to False, the returned list contains the apps' names. + If set to False, the returned list contains the models' names. If set to True, the list contains model info dictionaries. Returns diff --git a/management/test/manager_test.py b/management/test/manager_test.py new file mode 100644 index 000000000..08dcd6b5c --- /dev/null +++ b/management/test/manager_test.py @@ -0,0 +1,71 @@ +import unittest +import sys +import os +cur_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.abspath('%s/../' % cur_dir)) +import clipper_manager + +class ClipperManagerTestCase(unittest.TestCase): + + @classmethod + def setUpClass(self): + self.clipper_inst = clipper_manager.Clipper("localhost") + self.clipper_inst.start() + self.model_name = "m1" + self.model_version1 = 1 + self.model_version2 = 2 + + @classmethod + def tearDownClass(self): + self.clipper_inst.stop_all() + + def test_external_models_register_correctly(self): + name = "m1" + version1 = 1 + tags = ["test"] + input_type = "doubles" + result = self.clipper_inst.register_external_model(self.model_name, self.model_version1, tags, input_type) + self.assertTrue(result) + registered_model_info = self.clipper_inst.get_model_info(self.model_name, self.model_version1) + self.assertIsNotNone(registered_model_info) + + version2 = 2 + result = self.clipper_inst.register_external_model(self.model_name, self.model_version2, tags, input_type) + self.assertTrue(result) + registered_model_info = self.clipper_inst.get_model_info(self.model_name, self.model_version2) + self.assertIsNotNone(registered_model_info) + + def test_application_registers_correctly(self): + app_name = "app1" + input_type = "doubles" + default_output = "DEFAULT" + slo_micros = 30000 + self.clipper_inst.register_application(app_name, self.model_name, input_type, default_output, slo_micros) + registered_applications = self.clipper_inst.get_all_apps() + self.assertGreaterEqual(len(registered_applications), 1) + self.assertTrue(app_name in registered_applications) + + def test_add_container_for_external_model_fails(self): + result = self.clipper_inst.add_container(self.model_name, self.model_version1) + self.assertFalse(result) + + def test_model_version_sets_correctly(self): + self.clipper_inst.set_model_version(self.model_name, self.model_version1) + all_models = self.clipper_inst.get_all_models(verbose=True) + models_list_contains_correct_version = False + for model_info in all_models: + version = model_info["model_version"] + if version == self.model_version1: + models_list_contains_correct_version = True + self.assertTrue(model_info["is_current_version"]) + + self.assertTrue(models_list_contains_correct_version) + + +if __name__ == '__main__': + suite = unittest.TestSuite() + test_ordering = ['test_external_models_register_correctly', 'test_application_registers_correctly', + 'test_add_container_for_external_model_fails', 'test_model_version_sets_correctly'] + for test in test_ordering: + suite.addTest(ClipperManagerTestCase(test)) + unittest.TextTestRunner(verbosity=2).run(suite) \ No newline at end of file From 87b1a2f465fd0e514c920cd70adc807fdc39e6b8 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 03:39:10 -0600 Subject: [PATCH 09/36] More complete test suite for clipper manager --- bin/run_unittests.sh | 3 + management/test/clipper_manager_test.py | 222 ++++++++++++++++++++++++ management/test/manager_test.py | 71 -------- 3 files changed, 225 insertions(+), 71 deletions(-) create mode 100644 management/test/clipper_manager_test.py delete mode 100644 management/test/manager_test.py diff --git a/bin/run_unittests.sh b/bin/run_unittests.sh index 69de05fff..25965cccc 100755 --- a/bin/run_unittests.sh +++ b/bin/run_unittests.sh @@ -107,10 +107,13 @@ function run_libclipper_tests { function run_management_tests { echo -e "\nRunning management tests\n\n" ./src/management/managementtests --redis_port $REDIS_PORT + cd $DIR + python ../management/test/clipper_manager_test.py short } function run_frontend_tests { echo -e "\nRunning frontend tests\n\n" + cd $DIR ./src/frontends/frontendtests --redis_port $REDIS_PORT } diff --git a/management/test/clipper_manager_test.py b/management/test/clipper_manager_test.py new file mode 100644 index 000000000..b5af5460b --- /dev/null +++ b/management/test/clipper_manager_test.py @@ -0,0 +1,222 @@ +import unittest +import sys +import os +import json +import time +import requests +from sklearn import svm +cur_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.abspath('%s/../' % cur_dir)) +import clipper_manager + +class ClipperManagerTestCaseShort(unittest.TestCase): + + @classmethod + def setUpClass(self): + self.clipper_inst = clipper_manager.Clipper("localhost") + self.clipper_inst.start() + self.model_name = "m1" + self.model_version_1 = 1 + self.model_version_2 = 2 + self.deploy_model_name = "m3" + self.deploy_model_version = 1 + + @classmethod + def tearDownClass(self): + self.clipper_inst.stop_all() + + def test_external_models_register_correctly(self): + name = "m1" + version1 = 1 + tags = ["test"] + input_type = "doubles" + result = self.clipper_inst.register_external_model(self.model_name, self.model_version_1, tags, input_type) + self.assertTrue(result) + registered_model_info = self.clipper_inst.get_model_info(self.model_name, self.model_version_1) + self.assertIsNotNone(registered_model_info) + + version2 = 2 + result = self.clipper_inst.register_external_model(self.model_name, self.model_version_2, tags, input_type) + self.assertTrue(result) + registered_model_info = self.clipper_inst.get_model_info(self.model_name, self.model_version_2) + self.assertIsNotNone(registered_model_info) + + def test_application_registers_correctly(self): + app_name = "app1" + input_type = "doubles" + default_output = "DEFAULT" + slo_micros = 30000 + self.clipper_inst.register_application(app_name, self.model_name, input_type, default_output, slo_micros) + registered_applications = self.clipper_inst.get_all_apps() + self.assertGreaterEqual(len(registered_applications), 1) + self.assertTrue(app_name in registered_applications) + + def test_add_container_for_external_model_fails(self): + result = self.clipper_inst.add_container(self.model_name, self.model_version_1) + self.assertFalse(result) + + def test_model_version_sets_correctly(self): + self.clipper_inst.set_model_version(self.model_name, self.model_version_1) + all_models = self.clipper_inst.get_all_models(verbose=True) + models_list_contains_correct_version = False + for model_info in all_models: + version = model_info["model_version"] + if version == self.model_version_1: + models_list_contains_correct_version = True + self.assertTrue(model_info["is_current_version"]) + + self.assertTrue(models_list_contains_correct_version) + + def test_get_logs_creates_log_files(self): + log_file_names = self.clipper_inst.get_clipper_logs() + self.assertIsNotNone(log_file_names) + self.assertGreaterEqual(len(log_file_names), 1) + for file_name in log_file_names: + self.assertTrue(os.path.isfile(file_name)) + + def test_inspect_instance_returns_json_dict(self): + metrics = self.clipper_inst.inspect_instance() + self.assertEqual(type(metrics), dict) + self.assertGreaterEqual(len(metrics), 1) + + def test_model_deploys_successfully(self): + # Initialize a support vector classifier + # that will be deployed to a no-op container + model_data = svm.SVC() + container_name = "clipper/noop-container" + labels = ["test"] + input_type = "doubles" + result = self.clipper_inst.deploy_model(self.deploy_model_name, self.deploy_model_version, model_data, container_name, labels, input_type) + self.assertTrue(result) + model_info = self.clipper_inst.get_model_info(self.deploy_model_name, self.deploy_model_version) + self.assertIsNotNone(model_info) + running_containers_output = self.clipper_inst._execute_standard("docker ps -q --filter \"ancestor=clipper/noop-container\"") + self.assertIsNotNone(running_containers_output) + self.assertGreaterEqual(len(running_containers_output), 1) + + def test_add_container_for_deployed_model_succeeds(self): + result = self.clipper_inst.add_container(self.deploy_model_name, self.deploy_model_version) + self.assertTrue(result) + running_containers_output = self.clipper_inst._execute_standard("docker ps -q --filter \"ancestor=clipper/noop-container\"") + self.assertIsNotNone(running_containers_output) + split_output = running_containers_output.split("\n") + self.assertGreaterEqual(len(split_output), 2) + + def test_predict_function_deploys_successfully(self): + model_name = "m2" + model_version = 1 + predict_func = lambda inputs : ["0" for x in inputs] + labels = ["test"] + input_type = "doubles" + result = self.clipper_inst.deploy_predict_function(model_name, model_version, predict_func, labels, input_type) + self.assertTrue(result) + model_info = self.clipper_inst.get_model_info(model_name, model_version) + self.assertIsNotNone(model_info) + running_containers_output = self.clipper_inst._execute_standard("docker ps -q --filter \"ancestor=clipper/python-container\"") + self.assertIsNotNone(running_containers_output) + self.assertGreaterEqual(len(running_containers_output), 1) + + +class ClipperManagerTestCaseLong(unittest.TestCase): + + @classmethod + def setUpClass(self): + self.clipper_inst = clipper_manager.Clipper("localhost") + self.clipper_inst.start() + self.app_name_1 = "app1" + self.app_name_2 = "app2" + self.model_name_1 = "m1" + self.model_name_2 = "m2" + self.input_type = "doubles" + self.default_output = "DEFAULT" + self.latency_slo_micros = 30000 + self.clipper_inst.register_application(self.app_name_1, self.model_name_1, self.input_type, self.default_output, self.latency_slo_micros) + self.clipper_inst.register_application(self.app_name_2, self.model_name_2, self.input_type, self.default_output, self.latency_slo_micros) + + @classmethod + def tearDownClass(self): + self.clipper_inst.stop_all() + + def test_deployed_model_queried_successfully(self): + model_version = 1 + # Initialize a support vector classifier + # that will be deployed to a no-op container + model_data = svm.SVC() + container_name = "clipper/noop-container" + labels = ["test"] + result = self.clipper_inst.deploy_model(self.model_name_2, model_version, model_data, container_name, labels, self.input_type) + self.assertTrue(result) + + time.sleep(30) + + url = "http://localhost:1337/{}/predict".format(self.app_name_2) + test_input = [99.3, 18.9, 67.2, 34.2] + req_json = json.dumps({'uid': 0, 'input': test_input}) + headers = {'Content-type': 'application/json'} + response = requests.post(url, headers=headers, data=req_json) + parsed_response = json.loads(response.text) + self.assertNotEqual(parsed_response["output"], self.default_output) + + def test_deployed_predict_function_queried_successfully(self): + model_version = 1 + predict_func = lambda inputs : [str(len(x)) for x in inputs] + labels = ["test"] + input_type = "doubles" + result = self.clipper_inst.deploy_predict_function(self.model_name_1, model_version, predict_func, labels, input_type) + self.assertTrue(result) + + time.sleep(60) + + received_non_default_prediction = False + for i in range(0, 40): + url = "http://localhost:1337/{}/predict".format(self.app_name_1) + test_input = [101.1, 99.5, 107.2] + req_json = json.dumps({'uid': 0, 'input': test_input}) + headers = {'Content-type': 'application/json'} + response = requests.post(url, headers=headers, data=req_json) + parsed_response = json.loads(response.text) + output = parsed_response["output"] + if output == self.default_output: + time.sleep(20) + else: + received_non_default_prediction = True + self.assertEqual(int(output), len(test_input)) + break + + self.assertTrue(received_non_default_prediction) + + +SHORT_ARGS = ['s', 'short'] +LONG_ARGS = ['l', 'long'] + +SHORT_TEST_ORDERING = ['test_external_models_register_correctly', 'test_application_registers_correctly', + 'test_add_container_for_external_model_fails', 'test_model_version_sets_correctly', 'test_get_logs_creates_log_files', + 'test_inspect_instance_returns_json_dict', 'test_model_deploys_successfully', 'test_add_container_for_deployed_model_succeeds', + 'test_predict_function_deploys_successfully'] + +LONG_TEST_ORDERING = ['test_deployed_model_queried_successfully', 'test_deployed_predict_function_queried_successfully'] + +if __name__ == '__main__': + run_short = True + run_long = True + if len(sys.argv) > 1: + if sys.argv[1] in SHORT_ARGS: + run_long = False + elif sys.argv[1] in LONG_ARGS: + run_short = False + else: + print("Correct parameter values are either 's'/'short' or 'l'/'long' indicating which subset of tests should be executed!") + raise + + suite = unittest.TestSuite() + + if run_short: + for test in SHORT_TEST_ORDERING: + suite.addTest(ClipperManagerTestCaseShort(test)) + + if run_long: + for test in LONG_TEST_ORDERING: + suite.addTest(ClipperManagerTestCaseLong(test)) + + result = unittest.TextTestRunner(verbosity=2).run(suite) + sys.exit(not result.wasSuccessful()) \ No newline at end of file diff --git a/management/test/manager_test.py b/management/test/manager_test.py deleted file mode 100644 index 08dcd6b5c..000000000 --- a/management/test/manager_test.py +++ /dev/null @@ -1,71 +0,0 @@ -import unittest -import sys -import os -cur_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.abspath('%s/../' % cur_dir)) -import clipper_manager - -class ClipperManagerTestCase(unittest.TestCase): - - @classmethod - def setUpClass(self): - self.clipper_inst = clipper_manager.Clipper("localhost") - self.clipper_inst.start() - self.model_name = "m1" - self.model_version1 = 1 - self.model_version2 = 2 - - @classmethod - def tearDownClass(self): - self.clipper_inst.stop_all() - - def test_external_models_register_correctly(self): - name = "m1" - version1 = 1 - tags = ["test"] - input_type = "doubles" - result = self.clipper_inst.register_external_model(self.model_name, self.model_version1, tags, input_type) - self.assertTrue(result) - registered_model_info = self.clipper_inst.get_model_info(self.model_name, self.model_version1) - self.assertIsNotNone(registered_model_info) - - version2 = 2 - result = self.clipper_inst.register_external_model(self.model_name, self.model_version2, tags, input_type) - self.assertTrue(result) - registered_model_info = self.clipper_inst.get_model_info(self.model_name, self.model_version2) - self.assertIsNotNone(registered_model_info) - - def test_application_registers_correctly(self): - app_name = "app1" - input_type = "doubles" - default_output = "DEFAULT" - slo_micros = 30000 - self.clipper_inst.register_application(app_name, self.model_name, input_type, default_output, slo_micros) - registered_applications = self.clipper_inst.get_all_apps() - self.assertGreaterEqual(len(registered_applications), 1) - self.assertTrue(app_name in registered_applications) - - def test_add_container_for_external_model_fails(self): - result = self.clipper_inst.add_container(self.model_name, self.model_version1) - self.assertFalse(result) - - def test_model_version_sets_correctly(self): - self.clipper_inst.set_model_version(self.model_name, self.model_version1) - all_models = self.clipper_inst.get_all_models(verbose=True) - models_list_contains_correct_version = False - for model_info in all_models: - version = model_info["model_version"] - if version == self.model_version1: - models_list_contains_correct_version = True - self.assertTrue(model_info["is_current_version"]) - - self.assertTrue(models_list_contains_correct_version) - - -if __name__ == '__main__': - suite = unittest.TestSuite() - test_ordering = ['test_external_models_register_correctly', 'test_application_registers_correctly', - 'test_add_container_for_external_model_fails', 'test_model_version_sets_correctly'] - for test in test_ordering: - suite.addTest(ClipperManagerTestCase(test)) - unittest.TextTestRunner(verbosity=2).run(suite) \ No newline at end of file From 049dcdcb90f8d0dc3640f7e55489c2d947c3704d Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 03:45:40 -0600 Subject: [PATCH 10/36] 2 more tests, run all clipper mgr unit tests --- bin/run_unittests.sh | 2 +- management/test/clipper_manager_test.py | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/bin/run_unittests.sh b/bin/run_unittests.sh index 25965cccc..08c2d2700 100755 --- a/bin/run_unittests.sh +++ b/bin/run_unittests.sh @@ -108,7 +108,7 @@ function run_management_tests { echo -e "\nRunning management tests\n\n" ./src/management/managementtests --redis_port $REDIS_PORT cd $DIR - python ../management/test/clipper_manager_test.py short + python ../management/test/clipper_manager_test.py } function run_frontend_tests { diff --git a/management/test/clipper_manager_test.py b/management/test/clipper_manager_test.py index b5af5460b..9d9b5cd17 100644 --- a/management/test/clipper_manager_test.py +++ b/management/test/clipper_manager_test.py @@ -15,6 +15,7 @@ class ClipperManagerTestCaseShort(unittest.TestCase): def setUpClass(self): self.clipper_inst = clipper_manager.Clipper("localhost") self.clipper_inst.start() + self.app_name = "app1" self.model_name = "m1" self.model_version_1 = 1 self.model_version_2 = 2 @@ -42,14 +43,22 @@ def test_external_models_register_correctly(self): self.assertIsNotNone(registered_model_info) def test_application_registers_correctly(self): - app_name = "app1" input_type = "doubles" default_output = "DEFAULT" slo_micros = 30000 - self.clipper_inst.register_application(app_name, self.model_name, input_type, default_output, slo_micros) + self.clipper_inst.register_application(self.app_name, self.model_name, input_type, default_output, slo_micros) registered_applications = self.clipper_inst.get_all_apps() self.assertGreaterEqual(len(registered_applications), 1) - self.assertTrue(app_name in registered_applications) + self.assertTrue(self.app_name in registered_applications) + + def get_app_info_for_registered_app_returns_info_dictionary(self): + result = self.clipper_inst.get_app_info(self.app_name) + self.assertIsNotNone(result) + self.assertEqual(type(result), dict) + + def get_app_info_for_nonexistent_app_returns_none(self): + result = self.clipper_inst.get_app_info("fake_app") + self.assertIsNone(result) def test_add_container_for_external_model_fails(self): result = self.clipper_inst.add_container(self.model_name, self.model_version_1) @@ -189,7 +198,8 @@ def test_deployed_predict_function_queried_successfully(self): SHORT_ARGS = ['s', 'short'] LONG_ARGS = ['l', 'long'] -SHORT_TEST_ORDERING = ['test_external_models_register_correctly', 'test_application_registers_correctly', +SHORT_TEST_ORDERING = ['test_external_models_register_correctly', 'test_application_registers_correctly', + 'get_app_info_for_registered_app_returns_info_dictionary', 'get_app_info_for_nonexistent_app_returns_none', 'test_add_container_for_external_model_fails', 'test_model_version_sets_correctly', 'test_get_logs_creates_log_files', 'test_inspect_instance_returns_json_dict', 'test_model_deploys_successfully', 'test_add_container_for_deployed_model_succeeds', 'test_predict_function_deploys_successfully'] From 471a2c4a2df545fb331ec8ba6cbfa586dcc5a148 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 03:47:23 -0600 Subject: [PATCH 11/36] Revert "Fix test case" This reverts commit aa9da7e87eb84e215504e5964c0636a48f7716e5. --- src/libclipper/test/selection_policies_test.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libclipper/test/selection_policies_test.cpp b/src/libclipper/test/selection_policies_test.cpp index 44d093685..92f0aede8 100644 --- a/src/libclipper/test/selection_policies_test.cpp +++ b/src/libclipper/test/selection_policies_test.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include using namespace clipper; @@ -32,9 +31,9 @@ TEST_F(DefaultOutputSelectionPolicyTest, 1000, DefaultOutputSelectionPolicy::get_name(), {}}; - ASSERT_THROW( - policy_.select_predict_tasks(nullptr, zero_candidate_models_query, 0), - NoModelsFoundError); + auto zero_models_tasks = + policy_.select_predict_tasks(nullptr, zero_candidate_models_query, 0); + EXPECT_EQ(zero_models_tasks.size(), (size_t)0); } TEST_F(DefaultOutputSelectionPolicyTest, From cb8f01ff8dd316d5e6a47be4bd49e2c4f9bc01de Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 03:47:40 -0600 Subject: [PATCH 12/36] Revert "Fix tests, update json schema" This reverts commit 51c82a8b5f9959559937843183e9a2fe4921353c. --- src/frontends/CMakeLists.txt | 2 +- src/frontends/src/query_frontend.hpp | 1 - src/frontends/src/query_frontend_tests.cpp | 4 +--- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/frontends/CMakeLists.txt b/src/frontends/CMakeLists.txt index b874199e5..d31660fc9 100644 --- a/src/frontends/CMakeLists.txt +++ b/src/frontends/CMakeLists.txt @@ -14,5 +14,5 @@ add_executable(frontendtests EXCLUDE_FROM_ALL src/frontend_tests.cpp src/query_frontend_tests.cpp ) -target_link_libraries(frontendtests clipper boost httpserver gtest gmock_main rapidjson cxxopts) +target_link_libraries(frontendtests clipper httpserver gtest gmock_main rapidjson cxxopts) add_dependencies(unittests frontendtests) diff --git a/src/frontends/src/query_frontend.hpp b/src/frontends/src/query_frontend.hpp index a35b487cc..d620290c0 100644 --- a/src/frontends/src/query_frontend.hpp +++ b/src/frontends/src/query_frontend.hpp @@ -379,7 +379,6 @@ class RequestHandler { * "query_id" := int, * "output" := float, * "default" := boolean - * "default_explanation" := string (optional) * } */ static const std::string get_prediction_response_content( diff --git a/src/frontends/src/query_frontend_tests.cpp b/src/frontends/src/query_frontend_tests.cpp index 17e8927d9..330977a63 100644 --- a/src/frontends/src/query_frontend_tests.cpp +++ b/src/frontends/src/query_frontend_tests.cpp @@ -1,8 +1,6 @@ #include #include -#include - #include #include #include @@ -20,7 +18,7 @@ class MockQueryProcessor { MockQueryProcessor() = default; boost::future predict(Query query) { Response response(query, 3, 5, Output("-1.0", {std::make_pair("m", 1)}), - false, boost::optional{}); + false); return boost::make_ready_future(response); } boost::future update(FeedbackQuery /*feedback*/) { From ffb25a74fe8f149cdde8f41e766ff444097a23f0 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 03:47:50 -0600 Subject: [PATCH 13/36] Revert "Remove msg_ from exception" This reverts commit 01ec13727eba105447d262b204c68c87a5c3a3a6. --- src/libclipper/include/clipper/exceptions.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libclipper/include/clipper/exceptions.hpp b/src/libclipper/include/clipper/exceptions.hpp index 831a54dbf..40baaf61d 100644 --- a/src/libclipper/include/clipper/exceptions.hpp +++ b/src/libclipper/include/clipper/exceptions.hpp @@ -19,6 +19,9 @@ class NoModelsFoundError : public std::runtime_error { public: NoModelsFoundError(); const char* what() const noexcept; + + private: + std::string msg_; }; } // namespace clipper From e75823da7df93411d5ca4a756b524d5fb74e4690 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 03:48:13 -0600 Subject: [PATCH 14/36] Revert "Add latency default expl, format" This reverts commit c85f2c06a41de8cfcea10b61962c38980c433d6a. --- src/libclipper/src/query_processor.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/libclipper/src/query_processor.cpp b/src/libclipper/src/query_processor.cpp index 26817dea1..fa31e7235 100644 --- a/src/libclipper/src/query_processor.cpp +++ b/src/libclipper/src/query_processor.cpp @@ -112,18 +112,11 @@ boost::future QueryProcessor::predict(Query query) { vector outputs; vector used_models; - bool all_tasks_timed_out = true; for (auto r = task_futures.begin(); r != task_futures.end(); ++r) { if ((*r).is_ready()) { outputs.push_back((*r).get()); - all_tasks_timed_out = false; } } - if (all_tasks_timed_out && !task_futures.empty() && !default_explanation) { - default_explanation = - "Failed to retrieve a prediction response within the specified " - "latency SLO"; - } std::pair final_output = current_policy->combine_predictions(selection_state, query, outputs); From 44459bc50a446f56814b5f1a981f5a0a1a64f9e5 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 03:48:21 -0600 Subject: [PATCH 15/36] Revert "Format code" This reverts commit 57b0e970e71027f1a63d35189e2f6d78b356d41c. --- src/frontends/src/query_frontend.hpp | 3 +-- src/libclipper/include/clipper/datatypes.hpp | 3 +-- src/libclipper/include/clipper/exceptions.hpp | 2 +- src/libclipper/src/datatypes.cpp | 2 +- src/libclipper/src/exceptions.cpp | 7 ++++--- src/libclipper/src/query_processor.cpp | 18 ++++++------------ src/libclipper/src/selection_policies.cpp | 2 +- 7 files changed, 15 insertions(+), 22 deletions(-) diff --git a/src/frontends/src/query_frontend.hpp b/src/frontends/src/query_frontend.hpp index d620290c0..ef825654d 100644 --- a/src/frontends/src/query_frontend.hpp +++ b/src/frontends/src/query_frontend.hpp @@ -403,8 +403,7 @@ class RequestHandler { } clipper::json::add_bool(json_response, PREDICTION_RESPONSE_KEY_USED_DEFAULT, query_response.output_is_default_); - if (query_response.output_is_default_ && - query_response.default_explanation_) { + if(query_response.output_is_default_ && query_response.default_explanation_) { clipper::json::add_string(json_response, PREDICTION_RESPONSE_KEY_DEFAULT_EXPLANATION, query_response.default_explanation_.get()); diff --git a/src/libclipper/include/clipper/datatypes.hpp b/src/libclipper/include/clipper/datatypes.hpp index a07448cac..64fc94f5c 100644 --- a/src/libclipper/include/clipper/datatypes.hpp +++ b/src/libclipper/include/clipper/datatypes.hpp @@ -232,8 +232,7 @@ class Response { ~Response() = default; Response(Query query, QueryId query_id, const long duration_micros, - Output output, const bool is_default, - const boost::optional default_explanation); + Output output, const bool is_default, const boost::optional default_explanation); // default copy constructors Response(const Response &) = default; diff --git a/src/libclipper/include/clipper/exceptions.hpp b/src/libclipper/include/clipper/exceptions.hpp index 40baaf61d..3c2d0febf 100644 --- a/src/libclipper/include/clipper/exceptions.hpp +++ b/src/libclipper/include/clipper/exceptions.hpp @@ -9,7 +9,7 @@ class PredictError : public std::runtime_error { public: PredictError(const std::string msg); - const char* what() const noexcept; + const char *what() const noexcept; private: const std::string msg_; diff --git a/src/libclipper/src/datatypes.cpp b/src/libclipper/src/datatypes.cpp index 34bebb03f..27c4ef0b9 100644 --- a/src/libclipper/src/datatypes.cpp +++ b/src/libclipper/src/datatypes.cpp @@ -312,7 +312,7 @@ Response::Response(Query query, QueryId query_id, const long duration_micros, duration_micros_(duration_micros), output_(std::move(output)), output_is_default_(output_is_default), - default_explanation_(default_explanation) {} + default_explanation_(default_explanation){} std::string Response::debug_string() const noexcept { std::string debug; diff --git a/src/libclipper/src/exceptions.cpp b/src/libclipper/src/exceptions.cpp index 497f33d87..70236fb3a 100644 --- a/src/libclipper/src/exceptions.cpp +++ b/src/libclipper/src/exceptions.cpp @@ -8,10 +8,11 @@ namespace clipper { PredictError::PredictError(const std::string msg) : std::runtime_error(msg), msg_(msg) {} -const char* PredictError::what() const noexcept { return msg_.c_str(); } +const char *PredictError::what() const noexcept { return msg_.c_str(); } -NoModelsFoundError::NoModelsFoundError() - : std::runtime_error("No models found for query") {} +NoModelsFoundError::NoModelsFoundError() : + std::runtime_error("No models found for query") { +} const char* NoModelsFoundError::what() const noexcept { return "No models found for query"; diff --git a/src/libclipper/src/query_processor.cpp b/src/libclipper/src/query_processor.cpp index fa31e7235..6ebfa9163 100644 --- a/src/libclipper/src/query_processor.cpp +++ b/src/libclipper/src/query_processor.cpp @@ -7,9 +7,9 @@ #include #define PROVIDES_EXECUTORS -#include #include #include +#include #include #include @@ -67,9 +67,8 @@ boost::future QueryProcessor::predict(Query query) { boost::optional default_explanation; std::vector tasks; try { - tasks = - current_policy->select_predict_tasks(selection_state, query, query_id); - } catch (const NoModelsFoundError& e) { + tasks = current_policy->select_predict_tasks(selection_state, query, query_id); + } catch(const NoModelsFoundError& e) { default_explanation = "No registered models found for query"; } @@ -101,8 +100,7 @@ boost::future QueryProcessor::predict(Query query) { boost::promise response_promise; auto response_future = response_promise.get_future(); - // NOTE: We capture the num_completed, completed_flag, and default_explanation - // variables + // NOTE: We capture the num_completed, completed_flag, and default_explanation variables // so that they outlive the composed_futures. response_ready_future.then([ this, query, query_id, response_promise = std::move(response_promise), @@ -128,12 +126,8 @@ boost::future QueryProcessor::predict(Query query) { end - query.create_time_) .count(); - Response response{query, - query_id, - duration_micros, - final_output.first, - final_output.second, - default_explanation}; + Response response{query, query_id, duration_micros, final_output.first, + final_output.second, default_explanation}; response_promise.set_value(response); }); return response_future; diff --git a/src/libclipper/src/selection_policies.cpp b/src/libclipper/src/selection_policies.cpp index a0b2b2289..74b405e80 100644 --- a/src/libclipper/src/selection_policies.cpp +++ b/src/libclipper/src/selection_policies.cpp @@ -7,11 +7,11 @@ #include #include -#include #include #include #include #include +#include namespace clipper { From c116a9f8d3f555a46e2fa22ffa007758bdbed845 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 03:48:40 -0600 Subject: [PATCH 16/36] Revert "Propagate default explanation to response" This reverts commit 4455ccb527c986cad3fdd66caeb1f0da42f5ca2a. --- src/frontends/src/query_frontend.hpp | 6 ------ src/libclipper/include/clipper/datatypes.hpp | 6 ++---- src/libclipper/src/datatypes.cpp | 3 +-- src/libclipper/src/query_processor.cpp | 5 ++--- 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/frontends/src/query_frontend.hpp b/src/frontends/src/query_frontend.hpp index ef825654d..e3bb3356e 100644 --- a/src/frontends/src/query_frontend.hpp +++ b/src/frontends/src/query_frontend.hpp @@ -39,7 +39,6 @@ const std::string GET_METRICS = "^/metrics$"; const char* PREDICTION_RESPONSE_KEY_QUERY_ID = "query_id"; const char* PREDICTION_RESPONSE_KEY_OUTPUT = "output"; const char* PREDICTION_RESPONSE_KEY_USED_DEFAULT = "default"; -const char* PREDICTION_RESPONSE_KEY_DEFAULT_EXPLANATION = "default_explanation"; const char* PREDICTION_ERROR_RESPONSE_KEY_ERROR = "error"; const char* PREDICTION_ERROR_RESPONSE_KEY_CAUSE = "cause"; @@ -403,11 +402,6 @@ class RequestHandler { } clipper::json::add_bool(json_response, PREDICTION_RESPONSE_KEY_USED_DEFAULT, query_response.output_is_default_); - if(query_response.output_is_default_ && query_response.default_explanation_) { - clipper::json::add_string(json_response, - PREDICTION_RESPONSE_KEY_DEFAULT_EXPLANATION, - query_response.default_explanation_.get()); - } std::string content = clipper::json::to_json_string(json_response); return content; } diff --git a/src/libclipper/include/clipper/datatypes.hpp b/src/libclipper/include/clipper/datatypes.hpp index 64fc94f5c..f7cc0a178 100644 --- a/src/libclipper/include/clipper/datatypes.hpp +++ b/src/libclipper/include/clipper/datatypes.hpp @@ -6,8 +6,6 @@ #include #include -#include - namespace clipper { using ByteBuffer = std::vector; @@ -232,7 +230,7 @@ class Response { ~Response() = default; Response(Query query, QueryId query_id, const long duration_micros, - Output output, const bool is_default, const boost::optional default_explanation); + Output output, const bool is_default, const std::string default_explanation); // default copy constructors Response(const Response &) = default; @@ -249,7 +247,7 @@ class Response { long duration_micros_; Output output_; bool output_is_default_; - boost::optional default_explanation_; + std::string default_explanation_; }; class Feedback { diff --git a/src/libclipper/src/datatypes.cpp b/src/libclipper/src/datatypes.cpp index 27c4ef0b9..a15d71d97 100644 --- a/src/libclipper/src/datatypes.cpp +++ b/src/libclipper/src/datatypes.cpp @@ -305,8 +305,7 @@ Query::Query(std::string label, long user_id, std::shared_ptr input, create_time_(std::chrono::high_resolution_clock::now()) {} Response::Response(Query query, QueryId query_id, const long duration_micros, - Output output, const bool output_is_default, - const boost::optional default_explanation) + Output output, const bool output_is_default, const std::string default_explanation) : query_(std::move(query)), query_id_(query_id), duration_micros_(duration_micros), diff --git a/src/libclipper/src/query_processor.cpp b/src/libclipper/src/query_processor.cpp index 6ebfa9163..ba1d2c103 100644 --- a/src/libclipper/src/query_processor.cpp +++ b/src/libclipper/src/query_processor.cpp @@ -9,7 +9,6 @@ #define PROVIDES_EXECUTORS #include #include -#include #include #include @@ -69,7 +68,7 @@ boost::future QueryProcessor::predict(Query query) { try { tasks = current_policy->select_predict_tasks(selection_state, query, query_id); } catch(const NoModelsFoundError& e) { - default_explanation = "No registered models found for query"; + default_explanation = "No models found for query"; } log_info_formatted(LOGGING_TAG_QUERY_PROCESSOR, "Found {} tasks", @@ -127,7 +126,7 @@ boost::future QueryProcessor::predict(Query query) { .count(); Response response{query, query_id, duration_micros, final_output.first, - final_output.second, default_explanation}; + final_output.second}; response_promise.set_value(response); }); return response_future; From e28793326ce01e9fc4590eb329c007c7b795a150 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 03:48:48 -0600 Subject: [PATCH 17/36] Revert "Infra for default explanations" This reverts commit 590666191a1000097812072b572189536fe261bf. --- src/libclipper/include/clipper/datatypes.hpp | 3 +-- src/libclipper/include/clipper/exceptions.hpp | 9 --------- src/libclipper/src/datatypes.cpp | 5 ++--- src/libclipper/src/exceptions.cpp | 8 -------- src/libclipper/src/query_processor.cpp | 13 ++++--------- src/libclipper/src/selection_policies.cpp | 2 -- 6 files changed, 7 insertions(+), 33 deletions(-) diff --git a/src/libclipper/include/clipper/datatypes.hpp b/src/libclipper/include/clipper/datatypes.hpp index f7cc0a178..46e91ffc2 100644 --- a/src/libclipper/include/clipper/datatypes.hpp +++ b/src/libclipper/include/clipper/datatypes.hpp @@ -230,7 +230,7 @@ class Response { ~Response() = default; Response(Query query, QueryId query_id, const long duration_micros, - Output output, const bool is_default, const std::string default_explanation); + Output output, const bool is_default); // default copy constructors Response(const Response &) = default; @@ -247,7 +247,6 @@ class Response { long duration_micros_; Output output_; bool output_is_default_; - std::string default_explanation_; }; class Feedback { diff --git a/src/libclipper/include/clipper/exceptions.hpp b/src/libclipper/include/clipper/exceptions.hpp index 3c2d0febf..c9d9c4304 100644 --- a/src/libclipper/include/clipper/exceptions.hpp +++ b/src/libclipper/include/clipper/exceptions.hpp @@ -15,15 +15,6 @@ class PredictError : public std::runtime_error { const std::string msg_; }; -class NoModelsFoundError : public std::runtime_error { - public: - NoModelsFoundError(); - const char* what() const noexcept; - - private: - std::string msg_; -}; - } // namespace clipper #endif // CLIPPER_EXCEPTIONS_HPP diff --git a/src/libclipper/src/datatypes.cpp b/src/libclipper/src/datatypes.cpp index a15d71d97..dc4fc4fac 100644 --- a/src/libclipper/src/datatypes.cpp +++ b/src/libclipper/src/datatypes.cpp @@ -305,13 +305,12 @@ Query::Query(std::string label, long user_id, std::shared_ptr input, create_time_(std::chrono::high_resolution_clock::now()) {} Response::Response(Query query, QueryId query_id, const long duration_micros, - Output output, const bool output_is_default, const std::string default_explanation) + Output output, const bool output_is_default) : query_(std::move(query)), query_id_(query_id), duration_micros_(duration_micros), output_(std::move(output)), - output_is_default_(output_is_default), - default_explanation_(default_explanation){} + output_is_default_(output_is_default) {} std::string Response::debug_string() const noexcept { std::string debug; diff --git a/src/libclipper/src/exceptions.cpp b/src/libclipper/src/exceptions.cpp index 70236fb3a..ce1e3dcae 100644 --- a/src/libclipper/src/exceptions.cpp +++ b/src/libclipper/src/exceptions.cpp @@ -9,12 +9,4 @@ PredictError::PredictError(const std::string msg) : std::runtime_error(msg), msg_(msg) {} const char *PredictError::what() const noexcept { return msg_.c_str(); } - -NoModelsFoundError::NoModelsFoundError() : - std::runtime_error("No models found for query") { -} - -const char* NoModelsFoundError::what() const noexcept { - return "No models found for query"; -} } diff --git a/src/libclipper/src/query_processor.cpp b/src/libclipper/src/query_processor.cpp index ba1d2c103..d689530ae 100644 --- a/src/libclipper/src/query_processor.cpp +++ b/src/libclipper/src/query_processor.cpp @@ -63,13 +63,8 @@ boost::future QueryProcessor::predict(Query query) { std::shared_ptr selection_state = current_policy->deserialize(*state_opt); - boost::optional default_explanation; - std::vector tasks; - try { - tasks = current_policy->select_predict_tasks(selection_state, query, query_id); - } catch(const NoModelsFoundError& e) { - default_explanation = "No models found for query"; - } + std::vector tasks = + current_policy->select_predict_tasks(selection_state, query, query_id); log_info_formatted(LOGGING_TAG_QUERY_PROCESSOR, "Found {} tasks", tasks.size()); @@ -99,12 +94,12 @@ boost::future QueryProcessor::predict(Query query) { boost::promise response_promise; auto response_future = response_promise.get_future(); - // NOTE: We capture the num_completed, completed_flag, and default_explanation variables + // NOTE: We capture the num_completed and completed_flag variables // so that they outlive the composed_futures. response_ready_future.then([ this, query, query_id, response_promise = std::move(response_promise), selection_state, current_policy, task_futures = std::move(task_futures), - num_completed, completed_flag, default_explanation + num_completed, completed_flag ](auto) mutable { vector outputs; diff --git a/src/libclipper/src/selection_policies.cpp b/src/libclipper/src/selection_policies.cpp index 74b405e80..6f8e48c9e 100644 --- a/src/libclipper/src/selection_policies.cpp +++ b/src/libclipper/src/selection_policies.cpp @@ -11,7 +11,6 @@ #include #include #include -#include namespace clipper { @@ -61,7 +60,6 @@ std::vector DefaultOutputSelectionPolicy::select_predict_tasks( log_error_formatted(LOGGING_TAG_SELECTION_POLICY, "No candidate models for query with label {}", query.label_); - throw NoModelsFoundError(); } else { if (num_candidate_models > 1) { log_error_formatted(LOGGING_TAG_SELECTION_POLICY, From 4ec5781f9e37266b57b607bb8ee99216a2493c8a Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 03:49:31 -0600 Subject: [PATCH 18/36] EOF space --- management/test/clipper_manager_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/test/clipper_manager_test.py b/management/test/clipper_manager_test.py index 9d9b5cd17..ad270d2e4 100644 --- a/management/test/clipper_manager_test.py +++ b/management/test/clipper_manager_test.py @@ -229,4 +229,4 @@ def test_deployed_predict_function_queried_successfully(self): suite.addTest(ClipperManagerTestCaseLong(test)) result = unittest.TextTestRunner(verbosity=2).run(suite) - sys.exit(not result.wasSuccessful()) \ No newline at end of file + sys.exit(not result.wasSuccessful()) From 0dd97849567179a68939990c6c6aab83e923698b Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 03:50:06 -0600 Subject: [PATCH 19/36] format code --- management/test/clipper_manager_test.py | 455 +++++++++++++----------- 1 file changed, 245 insertions(+), 210 deletions(-) diff --git a/management/test/clipper_manager_test.py b/management/test/clipper_manager_test.py index ad270d2e4..e8c530a34 100644 --- a/management/test/clipper_manager_test.py +++ b/management/test/clipper_manager_test.py @@ -9,224 +9,259 @@ sys.path.insert(0, os.path.abspath('%s/../' % cur_dir)) import clipper_manager -class ClipperManagerTestCaseShort(unittest.TestCase): - @classmethod - def setUpClass(self): - self.clipper_inst = clipper_manager.Clipper("localhost") - self.clipper_inst.start() - self.app_name = "app1" - self.model_name = "m1" - self.model_version_1 = 1 - self.model_version_2 = 2 - self.deploy_model_name = "m3" - self.deploy_model_version = 1 - - @classmethod - def tearDownClass(self): - self.clipper_inst.stop_all() - - def test_external_models_register_correctly(self): - name = "m1" - version1 = 1 - tags = ["test"] - input_type = "doubles" - result = self.clipper_inst.register_external_model(self.model_name, self.model_version_1, tags, input_type) - self.assertTrue(result) - registered_model_info = self.clipper_inst.get_model_info(self.model_name, self.model_version_1) - self.assertIsNotNone(registered_model_info) - - version2 = 2 - result = self.clipper_inst.register_external_model(self.model_name, self.model_version_2, tags, input_type) - self.assertTrue(result) - registered_model_info = self.clipper_inst.get_model_info(self.model_name, self.model_version_2) - self.assertIsNotNone(registered_model_info) - - def test_application_registers_correctly(self): - input_type = "doubles" - default_output = "DEFAULT" - slo_micros = 30000 - self.clipper_inst.register_application(self.app_name, self.model_name, input_type, default_output, slo_micros) - registered_applications = self.clipper_inst.get_all_apps() - self.assertGreaterEqual(len(registered_applications), 1) - self.assertTrue(self.app_name in registered_applications) - - def get_app_info_for_registered_app_returns_info_dictionary(self): - result = self.clipper_inst.get_app_info(self.app_name) - self.assertIsNotNone(result) - self.assertEqual(type(result), dict) - - def get_app_info_for_nonexistent_app_returns_none(self): - result = self.clipper_inst.get_app_info("fake_app") - self.assertIsNone(result) - - def test_add_container_for_external_model_fails(self): - result = self.clipper_inst.add_container(self.model_name, self.model_version_1) - self.assertFalse(result) - - def test_model_version_sets_correctly(self): - self.clipper_inst.set_model_version(self.model_name, self.model_version_1) - all_models = self.clipper_inst.get_all_models(verbose=True) - models_list_contains_correct_version = False - for model_info in all_models: - version = model_info["model_version"] - if version == self.model_version_1: - models_list_contains_correct_version = True - self.assertTrue(model_info["is_current_version"]) - - self.assertTrue(models_list_contains_correct_version) - - def test_get_logs_creates_log_files(self): - log_file_names = self.clipper_inst.get_clipper_logs() - self.assertIsNotNone(log_file_names) - self.assertGreaterEqual(len(log_file_names), 1) - for file_name in log_file_names: - self.assertTrue(os.path.isfile(file_name)) - - def test_inspect_instance_returns_json_dict(self): - metrics = self.clipper_inst.inspect_instance() - self.assertEqual(type(metrics), dict) - self.assertGreaterEqual(len(metrics), 1) - - def test_model_deploys_successfully(self): - # Initialize a support vector classifier - # that will be deployed to a no-op container - model_data = svm.SVC() - container_name = "clipper/noop-container" - labels = ["test"] - input_type = "doubles" - result = self.clipper_inst.deploy_model(self.deploy_model_name, self.deploy_model_version, model_data, container_name, labels, input_type) - self.assertTrue(result) - model_info = self.clipper_inst.get_model_info(self.deploy_model_name, self.deploy_model_version) - self.assertIsNotNone(model_info) - running_containers_output = self.clipper_inst._execute_standard("docker ps -q --filter \"ancestor=clipper/noop-container\"") - self.assertIsNotNone(running_containers_output) - self.assertGreaterEqual(len(running_containers_output), 1) - - def test_add_container_for_deployed_model_succeeds(self): - result = self.clipper_inst.add_container(self.deploy_model_name, self.deploy_model_version) - self.assertTrue(result) - running_containers_output = self.clipper_inst._execute_standard("docker ps -q --filter \"ancestor=clipper/noop-container\"") - self.assertIsNotNone(running_containers_output) - split_output = running_containers_output.split("\n") - self.assertGreaterEqual(len(split_output), 2) - - def test_predict_function_deploys_successfully(self): - model_name = "m2" - model_version = 1 - predict_func = lambda inputs : ["0" for x in inputs] - labels = ["test"] - input_type = "doubles" - result = self.clipper_inst.deploy_predict_function(model_name, model_version, predict_func, labels, input_type) - self.assertTrue(result) - model_info = self.clipper_inst.get_model_info(model_name, model_version) - self.assertIsNotNone(model_info) - running_containers_output = self.clipper_inst._execute_standard("docker ps -q --filter \"ancestor=clipper/python-container\"") - self.assertIsNotNone(running_containers_output) - self.assertGreaterEqual(len(running_containers_output), 1) +class ClipperManagerTestCaseShort(unittest.TestCase): + @classmethod + def setUpClass(self): + self.clipper_inst = clipper_manager.Clipper("localhost") + self.clipper_inst.start() + self.app_name = "app1" + self.model_name = "m1" + self.model_version_1 = 1 + self.model_version_2 = 2 + self.deploy_model_name = "m3" + self.deploy_model_version = 1 + + @classmethod + def tearDownClass(self): + self.clipper_inst.stop_all() + + def test_external_models_register_correctly(self): + name = "m1" + version1 = 1 + tags = ["test"] + input_type = "doubles" + result = self.clipper_inst.register_external_model( + self.model_name, self.model_version_1, tags, input_type) + self.assertTrue(result) + registered_model_info = self.clipper_inst.get_model_info( + self.model_name, self.model_version_1) + self.assertIsNotNone(registered_model_info) + + version2 = 2 + result = self.clipper_inst.register_external_model( + self.model_name, self.model_version_2, tags, input_type) + self.assertTrue(result) + registered_model_info = self.clipper_inst.get_model_info( + self.model_name, self.model_version_2) + self.assertIsNotNone(registered_model_info) + + def test_application_registers_correctly(self): + input_type = "doubles" + default_output = "DEFAULT" + slo_micros = 30000 + self.clipper_inst.register_application(self.app_name, self.model_name, + input_type, default_output, + slo_micros) + registered_applications = self.clipper_inst.get_all_apps() + self.assertGreaterEqual(len(registered_applications), 1) + self.assertTrue(self.app_name in registered_applications) + + def get_app_info_for_registered_app_returns_info_dictionary(self): + result = self.clipper_inst.get_app_info(self.app_name) + self.assertIsNotNone(result) + self.assertEqual(type(result), dict) + + def get_app_info_for_nonexistent_app_returns_none(self): + result = self.clipper_inst.get_app_info("fake_app") + self.assertIsNone(result) + + def test_add_container_for_external_model_fails(self): + result = self.clipper_inst.add_container(self.model_name, + self.model_version_1) + self.assertFalse(result) + + def test_model_version_sets_correctly(self): + self.clipper_inst.set_model_version(self.model_name, + self.model_version_1) + all_models = self.clipper_inst.get_all_models(verbose=True) + models_list_contains_correct_version = False + for model_info in all_models: + version = model_info["model_version"] + if version == self.model_version_1: + models_list_contains_correct_version = True + self.assertTrue(model_info["is_current_version"]) + + self.assertTrue(models_list_contains_correct_version) + + def test_get_logs_creates_log_files(self): + log_file_names = self.clipper_inst.get_clipper_logs() + self.assertIsNotNone(log_file_names) + self.assertGreaterEqual(len(log_file_names), 1) + for file_name in log_file_names: + self.assertTrue(os.path.isfile(file_name)) + + def test_inspect_instance_returns_json_dict(self): + metrics = self.clipper_inst.inspect_instance() + self.assertEqual(type(metrics), dict) + self.assertGreaterEqual(len(metrics), 1) + + def test_model_deploys_successfully(self): + # Initialize a support vector classifier + # that will be deployed to a no-op container + model_data = svm.SVC() + container_name = "clipper/noop-container" + labels = ["test"] + input_type = "doubles" + result = self.clipper_inst.deploy_model( + self.deploy_model_name, self.deploy_model_version, model_data, + container_name, labels, input_type) + self.assertTrue(result) + model_info = self.clipper_inst.get_model_info( + self.deploy_model_name, self.deploy_model_version) + self.assertIsNotNone(model_info) + running_containers_output = self.clipper_inst._execute_standard( + "docker ps -q --filter \"ancestor=clipper/noop-container\"") + self.assertIsNotNone(running_containers_output) + self.assertGreaterEqual(len(running_containers_output), 1) + + def test_add_container_for_deployed_model_succeeds(self): + result = self.clipper_inst.add_container(self.deploy_model_name, + self.deploy_model_version) + self.assertTrue(result) + running_containers_output = self.clipper_inst._execute_standard( + "docker ps -q --filter \"ancestor=clipper/noop-container\"") + self.assertIsNotNone(running_containers_output) + split_output = running_containers_output.split("\n") + self.assertGreaterEqual(len(split_output), 2) + + def test_predict_function_deploys_successfully(self): + model_name = "m2" + model_version = 1 + predict_func = lambda inputs: ["0" for x in inputs] + labels = ["test"] + input_type = "doubles" + result = self.clipper_inst.deploy_predict_function( + model_name, model_version, predict_func, labels, input_type) + self.assertTrue(result) + model_info = self.clipper_inst.get_model_info(model_name, + model_version) + self.assertIsNotNone(model_info) + running_containers_output = self.clipper_inst._execute_standard( + "docker ps -q --filter \"ancestor=clipper/python-container\"") + self.assertIsNotNone(running_containers_output) + self.assertGreaterEqual(len(running_containers_output), 1) class ClipperManagerTestCaseLong(unittest.TestCase): - - @classmethod - def setUpClass(self): - self.clipper_inst = clipper_manager.Clipper("localhost") - self.clipper_inst.start() - self.app_name_1 = "app1" - self.app_name_2 = "app2" - self.model_name_1 = "m1" - self.model_name_2 = "m2" - self.input_type = "doubles" - self.default_output = "DEFAULT" - self.latency_slo_micros = 30000 - self.clipper_inst.register_application(self.app_name_1, self.model_name_1, self.input_type, self.default_output, self.latency_slo_micros) - self.clipper_inst.register_application(self.app_name_2, self.model_name_2, self.input_type, self.default_output, self.latency_slo_micros) - - @classmethod - def tearDownClass(self): - self.clipper_inst.stop_all() - - def test_deployed_model_queried_successfully(self): - model_version = 1 - # Initialize a support vector classifier - # that will be deployed to a no-op container - model_data = svm.SVC() - container_name = "clipper/noop-container" - labels = ["test"] - result = self.clipper_inst.deploy_model(self.model_name_2, model_version, model_data, container_name, labels, self.input_type) - self.assertTrue(result) - - time.sleep(30) - - url = "http://localhost:1337/{}/predict".format(self.app_name_2) - test_input = [99.3, 18.9, 67.2, 34.2] - req_json = json.dumps({'uid': 0, 'input': test_input}) - headers = {'Content-type': 'application/json'} - response = requests.post(url, headers=headers, data=req_json) - parsed_response = json.loads(response.text) - self.assertNotEqual(parsed_response["output"], self.default_output) - - def test_deployed_predict_function_queried_successfully(self): - model_version = 1 - predict_func = lambda inputs : [str(len(x)) for x in inputs] - labels = ["test"] - input_type = "doubles" - result = self.clipper_inst.deploy_predict_function(self.model_name_1, model_version, predict_func, labels, input_type) - self.assertTrue(result) - - time.sleep(60) - - received_non_default_prediction = False - for i in range(0, 40): - url = "http://localhost:1337/{}/predict".format(self.app_name_1) - test_input = [101.1, 99.5, 107.2] - req_json = json.dumps({'uid': 0, 'input': test_input}) - headers = {'Content-type': 'application/json'} - response = requests.post(url, headers=headers, data=req_json) - parsed_response = json.loads(response.text) - output = parsed_response["output"] - if output == self.default_output: - time.sleep(20) - else: - received_non_default_prediction = True - self.assertEqual(int(output), len(test_input)) - break - - self.assertTrue(received_non_default_prediction) + @classmethod + def setUpClass(self): + self.clipper_inst = clipper_manager.Clipper("localhost") + self.clipper_inst.start() + self.app_name_1 = "app1" + self.app_name_2 = "app2" + self.model_name_1 = "m1" + self.model_name_2 = "m2" + self.input_type = "doubles" + self.default_output = "DEFAULT" + self.latency_slo_micros = 30000 + self.clipper_inst.register_application( + self.app_name_1, self.model_name_1, self.input_type, + self.default_output, self.latency_slo_micros) + self.clipper_inst.register_application( + self.app_name_2, self.model_name_2, self.input_type, + self.default_output, self.latency_slo_micros) + + @classmethod + def tearDownClass(self): + self.clipper_inst.stop_all() + + def test_deployed_model_queried_successfully(self): + model_version = 1 + # Initialize a support vector classifier + # that will be deployed to a no-op container + model_data = svm.SVC() + container_name = "clipper/noop-container" + labels = ["test"] + result = self.clipper_inst.deploy_model( + self.model_name_2, model_version, model_data, container_name, + labels, self.input_type) + self.assertTrue(result) + + time.sleep(30) + + url = "http://localhost:1337/{}/predict".format(self.app_name_2) + test_input = [99.3, 18.9, 67.2, 34.2] + req_json = json.dumps({'uid': 0, 'input': test_input}) + headers = {'Content-type': 'application/json'} + response = requests.post(url, headers=headers, data=req_json) + parsed_response = json.loads(response.text) + self.assertNotEqual(parsed_response["output"], self.default_output) + + def test_deployed_predict_function_queried_successfully(self): + model_version = 1 + predict_func = lambda inputs: [str(len(x)) for x in inputs] + labels = ["test"] + input_type = "doubles" + result = self.clipper_inst.deploy_predict_function( + self.model_name_1, model_version, predict_func, labels, input_type) + self.assertTrue(result) + + time.sleep(60) + + received_non_default_prediction = False + for i in range(0, 40): + url = "http://localhost:1337/{}/predict".format(self.app_name_1) + test_input = [101.1, 99.5, 107.2] + req_json = json.dumps({'uid': 0, 'input': test_input}) + headers = {'Content-type': 'application/json'} + response = requests.post(url, headers=headers, data=req_json) + parsed_response = json.loads(response.text) + output = parsed_response["output"] + if output == self.default_output: + time.sleep(20) + else: + received_non_default_prediction = True + self.assertEqual(int(output), len(test_input)) + break + + self.assertTrue(received_non_default_prediction) SHORT_ARGS = ['s', 'short'] LONG_ARGS = ['l', 'long'] -SHORT_TEST_ORDERING = ['test_external_models_register_correctly', 'test_application_registers_correctly', - 'get_app_info_for_registered_app_returns_info_dictionary', 'get_app_info_for_nonexistent_app_returns_none', - 'test_add_container_for_external_model_fails', 'test_model_version_sets_correctly', 'test_get_logs_creates_log_files', - 'test_inspect_instance_returns_json_dict', 'test_model_deploys_successfully', 'test_add_container_for_deployed_model_succeeds', - 'test_predict_function_deploys_successfully'] - -LONG_TEST_ORDERING = ['test_deployed_model_queried_successfully', 'test_deployed_predict_function_queried_successfully'] +SHORT_TEST_ORDERING = [ + 'test_external_models_register_correctly', + 'test_application_registers_correctly', + 'get_app_info_for_registered_app_returns_info_dictionary', + 'get_app_info_for_nonexistent_app_returns_none', + 'test_add_container_for_external_model_fails', + 'test_model_version_sets_correctly', 'test_get_logs_creates_log_files', + 'test_inspect_instance_returns_json_dict', + 'test_model_deploys_successfully', + 'test_add_container_for_deployed_model_succeeds', + 'test_predict_function_deploys_successfully' +] + +LONG_TEST_ORDERING = [ + 'test_deployed_model_queried_successfully', + 'test_deployed_predict_function_queried_successfully' +] if __name__ == '__main__': - run_short = True - run_long = True - if len(sys.argv) > 1: - if sys.argv[1] in SHORT_ARGS: - run_long = False - elif sys.argv[1] in LONG_ARGS: - run_short = False - else: - print("Correct parameter values are either 's'/'short' or 'l'/'long' indicating which subset of tests should be executed!") - raise - - suite = unittest.TestSuite() - - if run_short: - for test in SHORT_TEST_ORDERING: - suite.addTest(ClipperManagerTestCaseShort(test)) - - if run_long: - for test in LONG_TEST_ORDERING: - suite.addTest(ClipperManagerTestCaseLong(test)) - - result = unittest.TextTestRunner(verbosity=2).run(suite) - sys.exit(not result.wasSuccessful()) + run_short = True + run_long = True + if len(sys.argv) > 1: + if sys.argv[1] in SHORT_ARGS: + run_long = False + elif sys.argv[1] in LONG_ARGS: + run_short = False + else: + print( + "Correct parameter values are either 's'/'short' or 'l'/'long' indicating which subset of tests should be executed!" + ) + raise + + suite = unittest.TestSuite() + + if run_short: + for test in SHORT_TEST_ORDERING: + suite.addTest(ClipperManagerTestCaseShort(test)) + + if run_long: + for test in LONG_TEST_ORDERING: + suite.addTest(ClipperManagerTestCaseLong(test)) + + result = unittest.TextTestRunner(verbosity=2).run(suite) + sys.exit(not result.wasSuccessful()) From a4482c30c4b70aa4ce2e9011ab7dffaff9e4fa8a Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 04:05:45 -0600 Subject: [PATCH 20/36] Fix unittests script --- bin/run_unittests.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/run_unittests.sh b/bin/run_unittests.sh index 08c2d2700..852861d61 100755 --- a/bin/run_unittests.sh +++ b/bin/run_unittests.sh @@ -113,7 +113,6 @@ function run_management_tests { function run_frontend_tests { echo -e "\nRunning frontend tests\n\n" - cd $DIR ./src/frontends/frontendtests --redis_port $REDIS_PORT } From 8fcd5095149ee6dd7c4aa6192bc818eaf43bf2a2 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 23:12:31 -0600 Subject: [PATCH 21/36] Rename test file --- {management => clipper_admin}/test/clipper_manager_test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {management => clipper_admin}/test/clipper_manager_test.py (100%) diff --git a/management/test/clipper_manager_test.py b/clipper_admin/test/clipper_manager_test.py similarity index 100% rename from management/test/clipper_manager_test.py rename to clipper_admin/test/clipper_manager_test.py From 1b128eaa29faafde3643a13c5f3aa223c058a647 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 23:14:29 -0600 Subject: [PATCH 22/36] Fix unit tests script --- bin/run_unittests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run_unittests.sh b/bin/run_unittests.sh index 852861d61..48c754499 100755 --- a/bin/run_unittests.sh +++ b/bin/run_unittests.sh @@ -108,7 +108,7 @@ function run_management_tests { echo -e "\nRunning management tests\n\n" ./src/management/managementtests --redis_port $REDIS_PORT cd $DIR - python ../management/test/clipper_manager_test.py + python ../clipper_admin/test/clipper_manager_test.py } function run_frontend_tests { From a3c22a1b08d98d3ed592ba0f8c2f07e9689c8ee6 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Tue, 30 May 2017 23:24:03 -0600 Subject: [PATCH 23/36] Add 'all' argument --- bin/run_unittests.sh | 2 +- clipper_admin/test/clipper_manager_test.py | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/bin/run_unittests.sh b/bin/run_unittests.sh index 48c754499..5e1441015 100755 --- a/bin/run_unittests.sh +++ b/bin/run_unittests.sh @@ -108,7 +108,7 @@ function run_management_tests { echo -e "\nRunning management tests\n\n" ./src/management/managementtests --redis_port $REDIS_PORT cd $DIR - python ../clipper_admin/test/clipper_manager_test.py + python ../clipper_admin/test/clipper_manager_test.py all } function run_frontend_tests { diff --git a/clipper_admin/test/clipper_manager_test.py b/clipper_admin/test/clipper_manager_test.py index e8c530a34..6b51d7a12 100644 --- a/clipper_admin/test/clipper_manager_test.py +++ b/clipper_admin/test/clipper_manager_test.py @@ -220,6 +220,7 @@ def test_deployed_predict_function_queried_successfully(self): SHORT_ARGS = ['s', 'short'] LONG_ARGS = ['l', 'long'] +ALL_ARGS = ['a', 'all'] SHORT_TEST_ORDERING = [ 'test_external_models_register_correctly', @@ -242,16 +243,22 @@ def test_deployed_predict_function_queried_successfully(self): if __name__ == '__main__': run_short = True run_long = True + args_invalid = False if len(sys.argv) > 1: if sys.argv[1] in SHORT_ARGS: run_long = False elif sys.argv[1] in LONG_ARGS: run_short = False - else: - print( - "Correct parameter values are either 's'/'short' or 'l'/'long' indicating which subset of tests should be executed!" - ) - raise + elif sys.argv[1] not in ALL_ARGS: + args_invalid = True + else: + args_invalid = True + + if args_invalid: + print( + "Missing a parameter with value 's'/'short', 'l'/'long', or 'a'/'all' indicating which subset of tests should be executed!" + ) + raise suite = unittest.TestSuite() From 52f997a93dad38fb735980b7c97e315d49e6f8a0 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Wed, 31 May 2017 00:48:23 -0600 Subject: [PATCH 24/36] fix test --- clipper_admin/test/clipper_manager_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/clipper_admin/test/clipper_manager_test.py b/clipper_admin/test/clipper_manager_test.py index 6b51d7a12..ec861a31c 100644 --- a/clipper_admin/test/clipper_manager_test.py +++ b/clipper_admin/test/clipper_manager_test.py @@ -148,10 +148,10 @@ class ClipperManagerTestCaseLong(unittest.TestCase): def setUpClass(self): self.clipper_inst = clipper_manager.Clipper("localhost") self.clipper_inst.start() - self.app_name_1 = "app1" - self.app_name_2 = "app2" - self.model_name_1 = "m1" - self.model_name_2 = "m2" + self.app_name_1 = "app3" + self.app_name_2 = "app4" + self.model_name_1 = "m4" + self.model_name_2 = "m5" self.input_type = "doubles" self.default_output = "DEFAULT" self.latency_slo_micros = 30000 @@ -200,11 +200,11 @@ def test_deployed_predict_function_queried_successfully(self): time.sleep(60) received_non_default_prediction = False + url = "http://localhost:1337/{}/predict".format(self.app_name_1) + test_input = [101.1, 99.5, 107.2] + req_json = json.dumps({'uid': 0, 'input': test_input}) + headers = {'Content-type': 'application/json'} for i in range(0, 40): - url = "http://localhost:1337/{}/predict".format(self.app_name_1) - test_input = [101.1, 99.5, 107.2] - req_json = json.dumps({'uid': 0, 'input': test_input}) - headers = {'Content-type': 'application/json'} response = requests.post(url, headers=headers, data=req_json) parsed_response = json.loads(response.text) output = parsed_response["output"] From 484364f5db2c96d8214985b0482532ac870760ff Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Wed, 31 May 2017 00:54:39 -0600 Subject: [PATCH 25/36] format code, add description --- clipper_admin/test/clipper_manager_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/clipper_admin/test/clipper_manager_test.py b/clipper_admin/test/clipper_manager_test.py index ec861a31c..27ce0cecc 100644 --- a/clipper_admin/test/clipper_manager_test.py +++ b/clipper_admin/test/clipper_manager_test.py @@ -8,6 +8,12 @@ cur_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.abspath('%s/../' % cur_dir)) import clipper_manager +""" +Executes a test suite consisting of two separate cases: short tests and long tests. +Before each case, an instance of clipper_manager.Clipper is created. Tests +are then performed by invoking methods on this instance, often resulting +in the execution of docker commands. +""" class ClipperManagerTestCaseShort(unittest.TestCase): From 93d72a566442bbb6a5b7b52f79ca274cfb89c1e5 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Fri, 2 Jun 2017 08:35:45 -0600 Subject: [PATCH 26/36] Pluralize test folder --- clipper_admin/{test => tests}/clipper_manager_test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename clipper_admin/{test => tests}/clipper_manager_test.py (100%) diff --git a/clipper_admin/test/clipper_manager_test.py b/clipper_admin/tests/clipper_manager_test.py similarity index 100% rename from clipper_admin/test/clipper_manager_test.py rename to clipper_admin/tests/clipper_manager_test.py From ac74e023d0f663d30675dc1cd2f61d546398a28e Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Fri, 2 Jun 2017 12:07:33 -0600 Subject: [PATCH 27/36] Address comments --- bin/run_unittests.sh | 12 ++++++- clipper_admin/tests/clipper_manager_test.py | 36 +++++++++------------ 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/bin/run_unittests.sh b/bin/run_unittests.sh index 5e1441015..ddc67a8ce 100755 --- a/bin/run_unittests.sh +++ b/bin/run_unittests.sh @@ -107,8 +107,12 @@ function run_libclipper_tests { function run_management_tests { echo -e "\nRunning management tests\n\n" ./src/management/managementtests --redis_port $REDIS_PORT +} + +function run_clipper_admin_tests { + echo -e "Running clipper admin tests" cd $DIR - python ../clipper_admin/test/clipper_manager_test.py all + python ../clipper_admin/tests/clipper_manager_test.py all } function run_frontend_tests { @@ -129,6 +133,9 @@ function run_all_tests { redis-cli -p $REDIS_PORT "flushall" run_management_tests redis-cli -p $REDIS_PORT "flushall" + sleep 5 + run_clipper_admin_tests + redis-cli -p $REDIS_PORT "flushall" run_jvm_container_tests redis-cli -p $REDIS_PORT "flushall" run_rpc_container_tests @@ -153,6 +160,9 @@ case $args in -m | --management ) set_test_environment run_management_tests ;; + -c | --clipperadmin ) set_test_environment + run_clipper_admin_tests + ;; -f | --frontend ) set_test_environment run_frontend_tests ;; diff --git a/clipper_admin/tests/clipper_manager_test.py b/clipper_admin/tests/clipper_manager_test.py index 27ce0cecc..792597c40 100644 --- a/clipper_admin/tests/clipper_manager_test.py +++ b/clipper_admin/tests/clipper_manager_test.py @@ -5,6 +5,7 @@ import time import requests from sklearn import svm +from optparse import OptionParser cur_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.abspath('%s/../' % cur_dir)) import clipper_manager @@ -193,6 +194,7 @@ def test_deployed_model_queried_successfully(self): response = requests.post(url, headers=headers, data=req_json) parsed_response = json.loads(response.text) self.assertNotEqual(parsed_response["output"], self.default_output) + self.assertFalse(parsed_response["default"]) def test_deployed_predict_function_queried_successfully(self): model_version = 1 @@ -247,32 +249,24 @@ def test_deployed_predict_function_queried_successfully(self): ] if __name__ == '__main__': - run_short = True - run_long = True - args_invalid = False - if len(sys.argv) > 1: - if sys.argv[1] in SHORT_ARGS: - run_long = False - elif sys.argv[1] in LONG_ARGS: - run_short = False - elif sys.argv[1] not in ALL_ARGS: - args_invalid = True - else: - args_invalid = True - - if args_invalid: - print( - "Missing a parameter with value 's'/'short', 'l'/'long', or 'a'/'all' indicating which subset of tests should be executed!" - ) - raise + usage = "%prog [options] (default option is '-a/--all')" + parser = OptionParser(usage=usage) + parser.add_option("-s", "--short", action="store_true", dest="run_short", help="Run the short suite of test cases") + parser.add_option("-l", "--long", action="store_true", dest="run_long", help="Run the long suite of test cases") + parser.add_option("-a", "--all", action="store_true", dest="run_all", help="Run all test cases") + (options, args) = parser.parse_args() + + # If neither the short nor the long option is specified, + # we will run all tests + options.run_all = options.run_all or ((not options.run_short) and (not options.run_long)) suite = unittest.TestSuite() - if run_short: + if options.run_short or options.run_all: for test in SHORT_TEST_ORDERING: - suite.addTest(ClipperManagerTestCaseShort(test)) + suite.addTest(ClipperManagerTestCaseShort(test)) - if run_long: + if options.run_long or options.run_all: for test in LONG_TEST_ORDERING: suite.addTest(ClipperManagerTestCaseLong(test)) From f042f59f260eff12d88eec589f0819688693721d Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Fri, 2 Jun 2017 12:07:56 -0600 Subject: [PATCH 28/36] Format code --- clipper_admin/tests/clipper_manager_test.py | 26 +++++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/clipper_admin/tests/clipper_manager_test.py b/clipper_admin/tests/clipper_manager_test.py index 792597c40..b6fc9e705 100644 --- a/clipper_admin/tests/clipper_manager_test.py +++ b/clipper_admin/tests/clipper_manager_test.py @@ -251,20 +251,36 @@ def test_deployed_predict_function_queried_successfully(self): if __name__ == '__main__': usage = "%prog [options] (default option is '-a/--all')" parser = OptionParser(usage=usage) - parser.add_option("-s", "--short", action="store_true", dest="run_short", help="Run the short suite of test cases") - parser.add_option("-l", "--long", action="store_true", dest="run_long", help="Run the long suite of test cases") - parser.add_option("-a", "--all", action="store_true", dest="run_all", help="Run all test cases") + parser.add_option( + "-s", + "--short", + action="store_true", + dest="run_short", + help="Run the short suite of test cases") + parser.add_option( + "-l", + "--long", + action="store_true", + dest="run_long", + help="Run the long suite of test cases") + parser.add_option( + "-a", + "--all", + action="store_true", + dest="run_all", + help="Run all test cases") (options, args) = parser.parse_args() # If neither the short nor the long option is specified, # we will run all tests - options.run_all = options.run_all or ((not options.run_short) and (not options.run_long)) + options.run_all = options.run_all or ((not options.run_short) and + (not options.run_long)) suite = unittest.TestSuite() if options.run_short or options.run_all: for test in SHORT_TEST_ORDERING: - suite.addTest(ClipperManagerTestCaseShort(test)) + suite.addTest(ClipperManagerTestCaseShort(test)) if options.run_long or options.run_all: for test in LONG_TEST_ORDERING: From 66fb3bb22082d8b59ccb935f42608453d08ac04e Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Fri, 2 Jun 2017 21:33:49 -0700 Subject: [PATCH 29/36] Replace optparse with argparse --- clipper_admin/tests/clipper_manager_test.py | 23 ++++++++++----------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/clipper_admin/tests/clipper_manager_test.py b/clipper_admin/tests/clipper_manager_test.py index b6fc9e705..16c406bf2 100644 --- a/clipper_admin/tests/clipper_manager_test.py +++ b/clipper_admin/tests/clipper_manager_test.py @@ -5,7 +5,7 @@ import time import requests from sklearn import svm -from optparse import OptionParser +from argparse import ArgumentParser cur_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.abspath('%s/../' % cur_dir)) import clipper_manager @@ -249,40 +249,39 @@ def test_deployed_predict_function_queried_successfully(self): ] if __name__ == '__main__': - usage = "%prog [options] (default option is '-a/--all')" - parser = OptionParser(usage=usage) - parser.add_option( + description = "Runs clipper manager tests. If no arguments are specified, all tests are executed." + parser = ArgumentParser(description) + parser.add_argument( "-s", "--short", action="store_true", dest="run_short", help="Run the short suite of test cases") - parser.add_option( + parser.add_argument( "-l", "--long", action="store_true", dest="run_long", help="Run the long suite of test cases") - parser.add_option( + parser.add_argument( "-a", "--all", action="store_true", dest="run_all", help="Run all test cases") - (options, args) = parser.parse_args() + args = parser.parse_args() - # If neither the short nor the long option is specified, + # If neither the short nor the long argument is specified, # we will run all tests - options.run_all = options.run_all or ((not options.run_short) and - (not options.run_long)) + args.run_all = args.run_all or ((not args.run_short) and (not args.run_long)) suite = unittest.TestSuite() - if options.run_short or options.run_all: + if args.run_short or args.run_all: for test in SHORT_TEST_ORDERING: suite.addTest(ClipperManagerTestCaseShort(test)) - if options.run_long or options.run_all: + if args.run_long or args.run_all: for test in LONG_TEST_ORDERING: suite.addTest(ClipperManagerTestCaseLong(test)) From 1abf588825941897851254b13d6c0a3caf14a587 Mon Sep 17 00:00:00 2001 From: Corey Zumar Date: Fri, 2 Jun 2017 21:34:17 -0700 Subject: [PATCH 30/36] format code --- clipper_admin/tests/clipper_manager_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clipper_admin/tests/clipper_manager_test.py b/clipper_admin/tests/clipper_manager_test.py index 16c406bf2..3ff5e0263 100644 --- a/clipper_admin/tests/clipper_manager_test.py +++ b/clipper_admin/tests/clipper_manager_test.py @@ -273,7 +273,8 @@ def test_deployed_predict_function_queried_successfully(self): # If neither the short nor the long argument is specified, # we will run all tests - args.run_all = args.run_all or ((not args.run_short) and (not args.run_long)) + args.run_all = args.run_all or ((not args.run_short) and + (not args.run_long)) suite = unittest.TestSuite() From 2731be5b155d61811172e5aa7566c560920d6029 Mon Sep 17 00:00:00 2001 From: Dan Crankshaw Date: Sat, 3 Jun 2017 09:22:05 -0700 Subject: [PATCH 31/36] clean up and commented out deploy_predict_function tests --- clipper_admin/tests/__init__.py | 0 clipper_admin/tests/clipper_manager_test.py | 100 +++++++++----------- 2 files changed, 47 insertions(+), 53 deletions(-) create mode 100644 clipper_admin/tests/__init__.py diff --git a/clipper_admin/tests/__init__.py b/clipper_admin/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clipper_admin/tests/clipper_manager_test.py b/clipper_admin/tests/clipper_manager_test.py index 3ff5e0263..e0a29583e 100644 --- a/clipper_admin/tests/clipper_manager_test.py +++ b/clipper_admin/tests/clipper_manager_test.py @@ -6,9 +6,7 @@ import requests from sklearn import svm from argparse import ArgumentParser -cur_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.abspath('%s/../' % cur_dir)) -import clipper_manager +from clipper_admin import clipper_manager """ Executes a test suite consisting of two separate cases: short tests and long tests. Before each case, an instance of clipper_manager.Clipper is created. Tests @@ -132,22 +130,22 @@ def test_add_container_for_deployed_model_succeeds(self): split_output = running_containers_output.split("\n") self.assertGreaterEqual(len(split_output), 2) - def test_predict_function_deploys_successfully(self): - model_name = "m2" - model_version = 1 - predict_func = lambda inputs: ["0" for x in inputs] - labels = ["test"] - input_type = "doubles" - result = self.clipper_inst.deploy_predict_function( - model_name, model_version, predict_func, labels, input_type) - self.assertTrue(result) - model_info = self.clipper_inst.get_model_info(model_name, - model_version) - self.assertIsNotNone(model_info) - running_containers_output = self.clipper_inst._execute_standard( - "docker ps -q --filter \"ancestor=clipper/python-container\"") - self.assertIsNotNone(running_containers_output) - self.assertGreaterEqual(len(running_containers_output), 1) + # def test_predict_function_deploys_successfully(self): + # model_name = "m2" + # model_version = 1 + # predict_func = lambda inputs: ["0" for x in inputs] + # labels = ["test"] + # input_type = "doubles" + # result = self.clipper_inst.deploy_predict_function( + # model_name, model_version, predict_func, labels, input_type) + # self.assertTrue(result) + # model_info = self.clipper_inst.get_model_info(model_name, + # model_version) + # self.assertIsNotNone(model_info) + # running_containers_output = self.clipper_inst._execute_standard( + # "docker ps -q --filter \"ancestor=clipper/python-container\"") + # self.assertIsNotNone(running_containers_output) + # self.assertGreaterEqual(len(running_containers_output), 1) class ClipperManagerTestCaseLong(unittest.TestCase): @@ -196,39 +194,35 @@ def test_deployed_model_queried_successfully(self): self.assertNotEqual(parsed_response["output"], self.default_output) self.assertFalse(parsed_response["default"]) - def test_deployed_predict_function_queried_successfully(self): - model_version = 1 - predict_func = lambda inputs: [str(len(x)) for x in inputs] - labels = ["test"] - input_type = "doubles" - result = self.clipper_inst.deploy_predict_function( - self.model_name_1, model_version, predict_func, labels, input_type) - self.assertTrue(result) - - time.sleep(60) - - received_non_default_prediction = False - url = "http://localhost:1337/{}/predict".format(self.app_name_1) - test_input = [101.1, 99.5, 107.2] - req_json = json.dumps({'uid': 0, 'input': test_input}) - headers = {'Content-type': 'application/json'} - for i in range(0, 40): - response = requests.post(url, headers=headers, data=req_json) - parsed_response = json.loads(response.text) - output = parsed_response["output"] - if output == self.default_output: - time.sleep(20) - else: - received_non_default_prediction = True - self.assertEqual(int(output), len(test_input)) - break - - self.assertTrue(received_non_default_prediction) - + # def test_deployed_predict_function_queried_successfully(self): + # model_version = 1 + # predict_func = lambda inputs: [str(len(x)) for x in inputs] + # labels = ["test"] + # input_type = "doubles" + # result = self.clipper_inst.deploy_predict_function( + # self.model_name_1, model_version, predict_func, labels, input_type) + # self.assertTrue(result) + # + # time.sleep(60) + # + # received_non_default_prediction = False + # url = "http://localhost:1337/{}/predict".format(self.app_name_1) + # test_input = [101.1, 99.5, 107.2] + # req_json = json.dumps({'uid': 0, 'input': test_input}) + # headers = {'Content-type': 'application/json'} + # for i in range(0, 40): + # response = requests.post(url, headers=headers, data=req_json) + # parsed_response = json.loads(response.text) + # output = parsed_response["output"] + # if output == self.default_output: + # time.sleep(20) + # else: + # received_non_default_prediction = True + # self.assertEqual(int(output), len(test_input)) + # break + # + # self.assertTrue(received_non_default_prediction) -SHORT_ARGS = ['s', 'short'] -LONG_ARGS = ['l', 'long'] -ALL_ARGS = ['a', 'all'] SHORT_TEST_ORDERING = [ 'test_external_models_register_correctly', @@ -240,12 +234,12 @@ def test_deployed_predict_function_queried_successfully(self): 'test_inspect_instance_returns_json_dict', 'test_model_deploys_successfully', 'test_add_container_for_deployed_model_succeeds', - 'test_predict_function_deploys_successfully' + # 'test_predict_function_deploys_successfully' ] LONG_TEST_ORDERING = [ 'test_deployed_model_queried_successfully', - 'test_deployed_predict_function_queried_successfully' + # 'test_deployed_predict_function_queried_successfully' ] if __name__ == '__main__': From 7f29196f85a83db75e171173bf94718f45bcccb6 Mon Sep 17 00:00:00 2001 From: Dan Crankshaw Date: Sat, 3 Jun 2017 09:32:43 -0700 Subject: [PATCH 32/36] format --- clipper_admin/tests/clipper_manager_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clipper_admin/tests/clipper_manager_test.py b/clipper_admin/tests/clipper_manager_test.py index e0a29583e..fd5698836 100644 --- a/clipper_admin/tests/clipper_manager_test.py +++ b/clipper_admin/tests/clipper_manager_test.py @@ -230,7 +230,8 @@ def test_deployed_model_queried_successfully(self): 'get_app_info_for_registered_app_returns_info_dictionary', 'get_app_info_for_nonexistent_app_returns_none', 'test_add_container_for_external_model_fails', - 'test_model_version_sets_correctly', 'test_get_logs_creates_log_files', + 'test_model_version_sets_correctly', + 'test_get_logs_creates_log_files', 'test_inspect_instance_returns_json_dict', 'test_model_deploys_successfully', 'test_add_container_for_deployed_model_succeeds', From dfeebb5f3976eb5f5719151258d98fa4fc3b30ef Mon Sep 17 00:00:00 2001 From: Dan Crankshaw Date: Sat, 3 Jun 2017 10:09:30 -0700 Subject: [PATCH 33/36] fixed import --- clipper_admin/tests/clipper_manager_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clipper_admin/tests/clipper_manager_test.py b/clipper_admin/tests/clipper_manager_test.py index fd5698836..557752d5a 100644 --- a/clipper_admin/tests/clipper_manager_test.py +++ b/clipper_admin/tests/clipper_manager_test.py @@ -6,7 +6,9 @@ import requests from sklearn import svm from argparse import ArgumentParser -from clipper_admin import clipper_manager +cur_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.abspath('%s/../' % cur_dir)) +import clipper_manager """ Executes a test suite consisting of two separate cases: short tests and long tests. Before each case, an instance of clipper_manager.Clipper is created. Tests From aff454b8904307901da262a07d82f24abfc3886f Mon Sep 17 00:00:00 2001 From: Dan Crankshaw Date: Sat, 3 Jun 2017 10:32:13 -0700 Subject: [PATCH 34/36] fixed unittests script --- bin/run_unittests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run_unittests.sh b/bin/run_unittests.sh index ddc67a8ce..b019657b1 100755 --- a/bin/run_unittests.sh +++ b/bin/run_unittests.sh @@ -112,7 +112,7 @@ function run_management_tests { function run_clipper_admin_tests { echo -e "Running clipper admin tests" cd $DIR - python ../clipper_admin/tests/clipper_manager_test.py all + python ../clipper_admin/tests/clipper_manager_test.py } function run_frontend_tests { From 4f39f2042842dbe1193434c60580a27bc3c83fd0 Mon Sep 17 00:00:00 2001 From: Dan Crankshaw Date: Sat, 3 Jun 2017 10:52:37 -0700 Subject: [PATCH 35/36] made unit tests use random redis port --- clipper_admin/tests/clipper_manager_test.py | 22 +++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/clipper_admin/tests/clipper_manager_test.py b/clipper_admin/tests/clipper_manager_test.py index 557752d5a..fd45548e7 100644 --- a/clipper_admin/tests/clipper_manager_test.py +++ b/clipper_admin/tests/clipper_manager_test.py @@ -9,6 +9,9 @@ cur_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.abspath('%s/../' % cur_dir)) import clipper_manager +import random +import socket + """ Executes a test suite consisting of two separate cases: short tests and long tests. Before each case, an instance of clipper_manager.Clipper is created. Tests @@ -16,11 +19,26 @@ in the execution of docker commands. """ +# range of ports where available ports can be found +PORT_RANGE = [34256, 40000] + +def find_unbound_port(): + """ + Returns an unbound port number on 127.0.0.1. + """ + while True: + port = random.randint(*PORT_RANGE) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + sock.bind(("127.0.0.1", port)) + return port + except socket.error: + print("randomly generated port %d is bound. Trying again." % port) class ClipperManagerTestCaseShort(unittest.TestCase): @classmethod def setUpClass(self): - self.clipper_inst = clipper_manager.Clipper("localhost") + self.clipper_inst = clipper_manager.Clipper("localhost", redis_port=find_unbound_port()) self.clipper_inst.start() self.app_name = "app1" self.model_name = "m1" @@ -153,7 +171,7 @@ def test_add_container_for_deployed_model_succeeds(self): class ClipperManagerTestCaseLong(unittest.TestCase): @classmethod def setUpClass(self): - self.clipper_inst = clipper_manager.Clipper("localhost") + self.clipper_inst = clipper_manager.Clipper("localhost", redis_port=find_unbound_port()) self.clipper_inst.start() self.app_name_1 = "app3" self.app_name_2 = "app4" From 5f192379b3e567abfc44727af330796b3918d564 Mon Sep 17 00:00:00 2001 From: Dan Crankshaw Date: Sat, 3 Jun 2017 10:57:50 -0700 Subject: [PATCH 36/36] format code --- clipper_admin/tests/clipper_manager_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/clipper_admin/tests/clipper_manager_test.py b/clipper_admin/tests/clipper_manager_test.py index fd45548e7..01ccecd46 100644 --- a/clipper_admin/tests/clipper_manager_test.py +++ b/clipper_admin/tests/clipper_manager_test.py @@ -11,7 +11,6 @@ import clipper_manager import random import socket - """ Executes a test suite consisting of two separate cases: short tests and long tests. Before each case, an instance of clipper_manager.Clipper is created. Tests @@ -22,6 +21,7 @@ # range of ports where available ports can be found PORT_RANGE = [34256, 40000] + def find_unbound_port(): """ Returns an unbound port number on 127.0.0.1. @@ -35,10 +35,12 @@ def find_unbound_port(): except socket.error: print("randomly generated port %d is bound. Trying again." % port) + class ClipperManagerTestCaseShort(unittest.TestCase): @classmethod def setUpClass(self): - self.clipper_inst = clipper_manager.Clipper("localhost", redis_port=find_unbound_port()) + self.clipper_inst = clipper_manager.Clipper( + "localhost", redis_port=find_unbound_port()) self.clipper_inst.start() self.app_name = "app1" self.model_name = "m1" @@ -171,7 +173,8 @@ def test_add_container_for_deployed_model_succeeds(self): class ClipperManagerTestCaseLong(unittest.TestCase): @classmethod def setUpClass(self): - self.clipper_inst = clipper_manager.Clipper("localhost", redis_port=find_unbound_port()) + self.clipper_inst = clipper_manager.Clipper( + "localhost", redis_port=find_unbound_port()) self.clipper_inst.start() self.app_name_1 = "app3" self.app_name_2 = "app4"