diff --git a/PythonContainerDockerfile b/PythonContainerDockerfile index 6d7d84095..705584d32 100644 --- a/PythonContainerDockerfile +++ b/PythonContainerDockerfile @@ -3,7 +3,8 @@ FROM clipper/py-rpc:latest MAINTAINER Dan Crankshaw COPY containers/python/python_container.py containers/python/python_container_entry.sh /container/ -COPY clipper_admin/pywrencloudpickle.py containers/python/python_container_conda_deps.txt /lib/ +COPY containers/python/python_container_conda_deps.txt /lib/ +COPY clipper_admin/ /lib/clipper_admin/ RUN conda install -y --file /lib/python_container_conda_deps.txt diff --git a/clipper_admin/clipper_manager.py b/clipper_admin/clipper_manager.py index cd0cfbaf6..ea0036776 100644 --- a/clipper_admin/clipper_manager.py +++ b/clipper_admin/clipper_manager.py @@ -636,24 +636,29 @@ def centered_predict(inputs): if not os.path.exists(serialization_dir): os.makedirs(serialization_dir) - # Export Anaconda environment + # Attempt to export Anaconda environment environment_file_abs_path = os.path.join(serialization_dir, environment_fname) - process = subprocess.Popen( - "PIP_FORMAT=legacy conda env export >> {environment_file_abs_path}". - format(environment_file_abs_path=environment_file_abs_path), - shell=True) - process.wait() + conda_env_exported = self._export_conda_env(environment_file_abs_path) - # Confirm that packages installed through conda are solvable - # Write out conda and pip dependency files to be supplied to container - if not (self._check_and_write_dependencies( - environment_file_abs_path, serialization_dir, conda_dep_fname, - pip_dep_fname)): - return False + if conda_env_exported: + print("Anaconda environment found. Verifying packages.") - os.remove(environment_file_abs_path) - print("Supplied environment details") + # Confirm that packages installed through conda are solvable + # Write out conda and pip dependency files to be supplied to container + if not (self._check_and_write_dependencies( + environment_file_abs_path, serialization_dir, + conda_dep_fname, pip_dep_fname)): + return False + + print("Supplied environment details") + else: + print( + "Warning: Anaconda environment was either not found or exporting the environment " + "failed. Your function will still be serialized deployed, but may fail due to " + "missing dependencies. In this case, please re-run inside an Anaconda environment. " + "See http://clipper.ai/documentation/python_model_deployment/ for more information." + ) # Write out function serialization func_file_path = os.path.join(serialization_dir, predict_fname) @@ -662,9 +667,13 @@ def centered_predict(inputs): print("Serialized and supplied predict function") # Deploy function - return self.deploy_model(name, version, serialization_dir, - default_python_container, labels, input_type, - num_containers) + deploy_result = self.deploy_model(name, version, serialization_dir, + default_python_container, labels, + input_type, num_containers) + # Remove temp files + shutil.rmtree(serialization_dir) + + return deploy_result def get_all_models(self, verbose=False): """Gets information about all models registered with Clipper. @@ -818,6 +827,24 @@ def inspect_selection_policy(self, app_name, uid): r = requests.post(url, headers=headers, data=req_json) return r.text + def _export_conda_env(self, environment_file_abs_path): + """Returns true if attempt to export the current conda environment is successful + + Parameters + ---------- + environment_file_abs_path : str + The desired absolute path for the exported conda environment file + """ + + process = subprocess.Popen( + "PIP_FORMAT=legacy conda env export >> {environment_file_abs_path}". + format(environment_file_abs_path=environment_file_abs_path), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True) + process.wait() + return process.returncode == 0 + def _check_and_write_dependencies(self, environment_path, directory, conda_dep_fname, pip_dep_fname): """Returns true if the provided conda environment is compatible with the container os. diff --git a/clipper_admin/tests/clipper_manager_test.py b/clipper_admin/tests/clipper_manager_test.py index 01ccecd46..da0cb4493 100644 --- a/clipper_admin/tests/clipper_manager_test.py +++ b/clipper_admin/tests/clipper_manager_test.py @@ -152,22 +152,22 @@ def test_add_container_for_deployed_model_succeeds(self): split_output = running_containers_output.split("\n") self.assertGreaterEqual(len(split_output), 2) - # def test_predict_function_deploys_successfully(self): - # model_name = "m2" - # model_version = 1 - # predict_func = lambda inputs: ["0" for x in inputs] - # labels = ["test"] - # input_type = "doubles" - # result = self.clipper_inst.deploy_predict_function( - # model_name, model_version, predict_func, labels, input_type) - # self.assertTrue(result) - # model_info = self.clipper_inst.get_model_info(model_name, - # model_version) - # self.assertIsNotNone(model_info) - # running_containers_output = self.clipper_inst._execute_standard( - # "docker ps -q --filter \"ancestor=clipper/python-container\"") - # self.assertIsNotNone(running_containers_output) - # self.assertGreaterEqual(len(running_containers_output), 1) + def test_predict_function_deploys_successfully(self): + model_name = "m2" + model_version = 1 + predict_func = lambda inputs: ["0" for x in inputs] + labels = ["test"] + input_type = "doubles" + result = self.clipper_inst.deploy_predict_function( + model_name, model_version, predict_func, labels, input_type) + self.assertTrue(result) + model_info = self.clipper_inst.get_model_info(model_name, + model_version) + self.assertIsNotNone(model_info) + running_containers_output = self.clipper_inst._execute_standard( + "docker ps -q --filter \"ancestor=clipper/python-container\"") + self.assertIsNotNone(running_containers_output) + self.assertGreaterEqual(len(running_containers_output), 1) class ClipperManagerTestCaseLong(unittest.TestCase): @@ -217,34 +217,34 @@ def test_deployed_model_queried_successfully(self): self.assertNotEqual(parsed_response["output"], self.default_output) self.assertFalse(parsed_response["default"]) - # def test_deployed_predict_function_queried_successfully(self): - # model_version = 1 - # predict_func = lambda inputs: [str(len(x)) for x in inputs] - # labels = ["test"] - # input_type = "doubles" - # result = self.clipper_inst.deploy_predict_function( - # self.model_name_1, model_version, predict_func, labels, input_type) - # self.assertTrue(result) - # - # time.sleep(60) - # - # received_non_default_prediction = False - # url = "http://localhost:1337/{}/predict".format(self.app_name_1) - # test_input = [101.1, 99.5, 107.2] - # req_json = json.dumps({'uid': 0, 'input': test_input}) - # headers = {'Content-type': 'application/json'} - # for i in range(0, 40): - # response = requests.post(url, headers=headers, data=req_json) - # parsed_response = json.loads(response.text) - # output = parsed_response["output"] - # if output == self.default_output: - # time.sleep(20) - # else: - # received_non_default_prediction = True - # self.assertEqual(int(output), len(test_input)) - # break - # - # self.assertTrue(received_non_default_prediction) + def test_deployed_predict_function_queried_successfully(self): + model_version = 1 + predict_func = lambda inputs: [str(len(x)) for x in inputs] + labels = ["test"] + input_type = "doubles" + result = self.clipper_inst.deploy_predict_function( + self.model_name_1, model_version, predict_func, labels, input_type) + self.assertTrue(result) + + time.sleep(60) + + received_non_default_prediction = False + url = "http://localhost:1337/{}/predict".format(self.app_name_1) + test_input = [101.1, 99.5, 107.2] + req_json = json.dumps({'uid': 0, 'input': test_input}) + headers = {'Content-type': 'application/json'} + for i in range(0, 40): + response = requests.post(url, headers=headers, data=req_json) + parsed_response = json.loads(response.text) + output = parsed_response["output"] + if output == self.default_output: + time.sleep(20) + else: + received_non_default_prediction = True + self.assertEqual(int(output), len(test_input)) + break + + self.assertTrue(received_non_default_prediction) SHORT_TEST_ORDERING = [ diff --git a/containers/python/python_container.py b/containers/python/python_container.py index 42a5fcb2b..b29525ddd 100644 --- a/containers/python/python_container.py +++ b/containers/python/python_container.py @@ -7,7 +7,7 @@ np.set_printoptions(threshold=np.nan) sys.path.append(os.path.abspath("/lib/")) -import pywrencloudpickle +from clipper_admin import pywrencloudpickle IMPORT_ERROR_RETURN_CODE = 3 @@ -118,5 +118,6 @@ def _log_incorrect_input_type(self, input_type): rpc_service = rpc.RPCService() rpc_service.start(model, ip, port, model_name, model_version, input_type) - except ImportError: + except ImportError as e: + print(e) sys.exit(IMPORT_ERROR_RETURN_CODE) diff --git a/containers/python/python_container_entry.sh b/containers/python/python_container_entry.sh index 15d672b99..be368bbe8 100755 --- a/containers/python/python_container_entry.sh +++ b/containers/python/python_container_entry.sh @@ -2,12 +2,40 @@ IMPORT_ERROR_RETURN_CODE=3 -echo "Attempting to run Python container without installing dependencies" -/bin/bash -c "exec python /container/python_container.py" +CONDA_DEPS_PATH="/model/conda_dependencies.txt" +PIP_DEPS_PATH="/model/pip_dependencies.txt" +CONTAINER_SCRIPT_PATH="/container/python_container.py" + +test -f $CONDA_DEPS_PATH +dependency_check_returncode=$? + +if [ $dependency_check_returncode -eq 0 ]; then + echo "First attempting to run Python container without installing supplied dependencies." +else + echo "No dependencies supplied. Attempting to run Python container." +fi + +/bin/bash -c "exec python $CONTAINER_SCRIPT_PATH" + if [ $? -eq $IMPORT_ERROR_RETURN_CODE ]; then - echo "Running Python container without installing dependencies fails" - echo "Will install dependencies and try again" - conda install -y --file /model/conda_dependencies.txt - pip install -r /model/pip_dependencies.txt - /bin/bash -c "exec python /container/python_container.py" + if [ $dependency_check_returncode -eq 0 ]; then + echo "Encountered an ImportError when running Python container without installing supplied dependencies." + echo "Will install supplied dependencies and try again." + conda install -y --file $CONDA_DEPS_PATH + pip install -r $PIP_DEPS_PATH + + /bin/bash -c "exec python $CONTAINER_SCRIPT_PATH" + + if [ $? -eq $IMPORT_ERROR_RETURN_CODE ]; then + echo "Encountered an ImportError even after installing supplied dependencies." + exit 1 + fi + else + echo "Encountered an ImportError when running Python container." + echo "Please supply necessary dependencies through an Anaconda environment and try again." + exit 1 + fi fi + +echo "Encountered error not related to missing packages. Please refer to the container log to diagnose." +exit 1