From ecaa99b65446467b823c8dd9a41103ecc3226806 Mon Sep 17 00:00:00 2001 From: GP Saggese Date: Mon, 2 Jun 2025 17:10:23 -0400 Subject: [PATCH 1/3] Improve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-commit checks: All checks passed ✅ --- .pre-commit-config.yaml | 19 +++++++++++++++++++ pyproject.toml | 27 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..a46f34397 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,19 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.4 + hooks: + - id: ruff + + - repo: local + hooks: + - id: fixit-lint + name: fixit lint + entry: fixit lint + language: system + types: [python] + + - id: pyrefly + name: pyrefly lint + entry: pyrefly lint + language: system + types: [python] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..3d8890ac2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,27 @@ +[tool.ruff] +line-length = 80 +target-version = "py311" +fix = true +exclude = [ + "**/outcomes/**", + "**/import_check/example/**" +] + +[tool.ruff.lint] +exclude = [ + "**/outcomes/**", + "**/import_check/example/**" +] + +[tool.pyrefly] +# Currently, Pyrefly doesn't officially support [tool.pyrefly] config +# Instead, use CLI flags like `pyrefly lint . --config path/to/config.yaml` +# But you can still define this for documentation or future support +path = "." + +[tool.fixit] +lint.ignore_codes = [] +lint.enabled = true +lint.format = "text" +lint.exclude = ["tests/fixtures"] +lint.show_source = true From ee96a139e408bd0ab9a116077a2a61e3076df02b Mon Sep 17 00:00:00 2001 From: GP Saggese Date: Mon, 2 Jun 2025 17:13:26 -0400 Subject: [PATCH 2/3] Lint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-commit checks: All checks passed ✅ --- config_root/config/config_.py | 16 +- config_root/config/config_builder.py | 6 +- config_root/config/config_list_builder.py | 7 +- config_root/config/test/test_config.py | 60 +- .../config/test/test_config_builder.py | 4 +- .../config/test/test_config_list_builder.py | 6 +- config_root/config/test/test_config_utils.py | 23 +- dev_scripts_helpers/aws/am_aws.py | 3 +- .../chatgpt/manage_chatgpt_assistant.py | 4 +- .../chatgpt/run_simple_chatgpt.py | 3 +- .../CmTask1292_Rename_old_aws_env_vars.py | 3 +- ...to_str_and_df_to_short_str_into_hpandas.py | 3 +- .../coding_tools/diff_to_vimdiff.py | 7 +- .../coding_tools/find_unused_golden_files.py | 4 +- dev_scripts_helpers/coding_tools/grsync.py | 13 +- .../invite_github_collaborator.py | 4 +- .../coding_tools/manage_cache.py | 3 +- .../coding_tools/measure_import_times.py | 7 +- .../coding_tools/parallel_script_template.py | 3 +- .../coding_tools/process_prof.py | 3 +- .../coding_tools/script_template.py | 3 +- .../coding_tools/toml_merge.py | 3 +- .../coding_tools/traceback_to_cfile.py | 3 +- .../coding_tools/transform_template.py | 3 +- dev_scripts_helpers/coding_tools/url.py | 3 +- .../dockerize/dockerized_template.py | 3 +- .../documentation/convert_docx_to_markdown.py | 3 +- .../documentation/dockerized_graphviz.py | 3 +- .../documentation/dockerized_latex.py | 3 +- .../documentation/dockerized_mermaid.py | 3 +- .../documentation/dockerized_pandoc.py | 7 +- .../documentation/dockerized_prettier.py | 3 +- .../dockerized_tikz_to_bitmap.py | 3 +- .../documentation/generate_script_catalog.py | 8 +- .../documentation/lint_notes.py | 6 +- .../documentation/notes_to_pdf.py | 20 +- .../documentation/preprocess_notes.py | 13 +- .../documentation/publish_notes.py | 3 +- .../documentation/render_images.py | 14 +- .../documentation/replace_latex.py | 11 +- .../documentation/run_pandoc.py | 3 +- .../test/test_dockerized_pandoc.py | 2 - .../test/test_dockerized_prettier.py | 1 - .../test_extract_headers_from_markdown.py | 1 - .../test/test_generate_readme_index.py | 6 +- .../documentation/test/test_lint_notes.py | 3 - .../documentation/test/test_notes_to_pdf.py | 1 - .../test/test_preprocess_notes.py | 1 - .../documentation/test/test_render_images.py | 4 - .../test/test_transform_notes.py | 1 - .../documentation/transform_notes.py | 3 +- .../encrypt_models/encrypt_model.py | 41 +- dev_scripts_helpers/git/gd_notebook.py | 7 +- .../git/git_hooks/install_hooks.py | 3 +- .../git/git_hooks/pre-commit-dry-run.py | 7 +- .../git/git_hooks/translate.py | 3 +- dev_scripts_helpers/git/git_submodules.py | 3 +- dev_scripts_helpers/git/gsp.py | 3 +- dev_scripts_helpers/git/gup.py | 3 +- .../github/dockerized_sync_gh_issue_labels.py | 4 +- .../dockerized_sync_gh_repo_settings.py | 11 +- .../github/set_secrets_and_variables.py | 3 +- .../github/sync_gh_issue_labels.py | 3 +- .../github/sync_gh_projects.py | 3 +- .../test_dockerized_sync_gh_issue_labels.py | 2 - .../github/test/test_sync_gh_issue_labels.py | 1 - dev_scripts_helpers/infra/gdrive.py | 3 +- dev_scripts_helpers/infra/old/ssh_tunnels.py | 3 +- dev_scripts_helpers/lib_tasks_data_qa.py | 8 +- dev_scripts_helpers/llms/ai_review.py | 1 - .../llms/dockerized_llm_review.py | 4 +- dev_scripts_helpers/llms/llm_apply_cfile.py | 4 +- dev_scripts_helpers/llms/llm_prompts.py | 13 +- dev_scripts_helpers/llms/llm_review.py | 8 +- dev_scripts_helpers/llms/llm_transform.py | 1 - .../llms/test/test_llm_prompts.py | 3 - .../llms/test/test_llm_transform.py | 3 +- dev_scripts_helpers/misc/extract_bounties.py | 3 +- dev_scripts_helpers/misc/get_url_titles.py | 1 - .../notebooks/add_toc_to_notebook.py | 3 +- .../dockerized_extract_notebook_images.py | 15 +- .../notebooks/extract_notebook_images.py | 3 +- .../notebooks/process_jupytext.py | 3 +- .../notebooks/publish_notebook.py | 4 +- .../notebooks/run_jupyter_server.py | 7 +- dev_scripts_helpers/notebooks/run_notebook.py | 7 +- ...test_dockerized_extract_notebook_images.py | 6 +- .../test/test_extract_notebook_images.py | 2 - .../old/create_conda/_setenv_amp.py | 3 +- .../old/create_conda/_setenv_lib.py | 8 +- .../old/create_conda/install/create_conda.py | 7 +- .../install/print_conda_packages.py | 3 +- dev_scripts_helpers/old/linter/linter.py | 17 +- .../old/linter/linter_master_report.py | 15 +- .../old/linter/pre_pr_checklist.py | 3 +- .../old/linter/process_jupytext.py | 3 +- .../poetry/run_poetry_debug.py | 3 +- .../filter_repo/git-filter-repo | 20 +- .../filter_repo/lint_history.py | 10 +- .../SorrTask396_scraping_script.ipynb | 85 +- .../system_tools/capture_notebook_cells.py | 18 +- .../system_tools/extract_cfile.py | 3 +- dev_scripts_helpers/system_tools/ffind.py | 3 +- dev_scripts_helpers/system_tools/jackdoc.py | 4 +- .../system_tools/remove_escape_chars.py | 3 +- .../system_tools/replace_text.py | 22 +- .../system_tools/save_screenshot.py | 7 +- dev_scripts_helpers/system_tools/tg.py | 3 +- dev_scripts_helpers/system_tools/zip_files.py | 3 +- .../test/test_find_unused_golden_files.py | 12 +- .../test/test_process_jupytext.py | 1 - .../test/test_publish_notebook.py | 1 + dev_scripts_helpers/test/test_replace_text.py | 10 +- dev_scripts_helpers/test/test_run_notebook.py | 4 +- dev_scripts_helpers/test/test_toml_merge.py | 6 +- dev_scripts_helpers/thin_client/build.py | 7 +- .../to_clean/gen_utils.ORIG.py | 31 +- dev_scripts_helpers/to_clean/gen_utils.py | 23 +- .../notebooks/CmampTask7292_ns_vs_us.ipynb | 6 +- .../notebooks/CmampTask7292_ns_vs_us.py | 6 +- .../Master_buildmeister_dashboard.ipynb | 751 +++++++-------- helpers/create_links.py | 11 +- helpers/hasyncio.py | 9 +- helpers/haws.py | 5 +- helpers/hcache.py | 12 +- helpers/hcache_simple.py | 5 +- helpers/hcfile.py | 1 - helpers/hchatgpt.py | 6 +- helpers/hcsv.py | 4 +- helpers/hdatetime.py | 8 +- helpers/hdbg.py | 10 +- helpers/hdict.py | 11 +- helpers/hdocker.py | 6 +- helpers/henv.py | 2 +- helpers/hgit.py | 6 +- helpers/hgoogle_drive_api.py | 4 +- helpers/hintrospection.py | 4 +- helpers/hio.py | 4 +- helpers/hjoblib.py | 4 +- helpers/hjupyter.py | 10 +- helpers/hlatex.py | 4 +- helpers/hlist.py | 1 + helpers/hlogging.py | 7 +- helpers/hmarkdown.py | 13 +- helpers/hnumba.py | 1 - helpers/hnumpy.py | 10 +- helpers/hopen.py | 4 +- helpers/hopenai.py | 28 +- helpers/hpandas.py | 21 +- helpers/hparquet.py | 4 +- helpers/hparser.py | 8 +- helpers/hpickle.py | 4 +- helpers/hplayback.py | 16 +- helpers/hprint.py | 10 +- helpers/hpytest.py | 4 +- helpers/hretry.py | 9 +- helpers/hs3.py | 8 +- helpers/hsecrets.py | 3 +- helpers/hsql.py | 4 +- helpers/hsql_implementation.py | 10 +- helpers/hsystem.py | 18 +- helpers/htest_logger.py | 4 +- helpers/htimer.py | 4 +- helpers/htqdm.py | 1 + helpers/htraceback.py | 2 +- helpers/htranslate.py | 1 + helpers/hunit_test.py | 17 +- helpers/hunit_test_utils.py | 8 +- helpers/lib_tasks.py | 2 +- helpers/lib_tasks_aws.py | 24 +- helpers/lib_tasks_docker.py | 8 +- helpers/lib_tasks_docker_release.py | 24 +- helpers/lib_tasks_find.py | 8 +- helpers/lib_tasks_gh.py | 11 +- helpers/lib_tasks_git.py | 8 +- helpers/lib_tasks_lint.py | 10 +- helpers/lib_tasks_perms.py | 4 +- helpers/lib_tasks_pytest.py | 12 +- helpers/logging_testing/logging_main.py | 2 +- .../Master.how_to_use_hgoogle_drive_api.ipynb | 854 +++++++++--------- .../Master.how_to_use_hgoogle_drive_api.py | 21 +- ...1_Playback_approach_for_unit_testing.ipynb | 2 - ...2291_Playback_approach_for_unit_testing.py | 3 - helpers/notebooks/cache.py | 1 + helpers/notebooks/gallery_s3.ipynb | 1 - helpers/notebooks/hopenai_tutorial.py | 12 +- helpers/old/tunnels.py | 3 +- helpers/old/user_credentials.py | 7 +- helpers/repo_config_utils.py | 1 - helpers/stage_linked_file.py | 4 +- helpers/telegram_notify/get_chat_id.py | 3 +- helpers/test/test_hcache.py | 16 +- helpers/test/test_hcache_simple.py | 20 +- helpers/test/test_hcfile.py | 3 - helpers/test/test_hdataframe.py | 5 - helpers/test/test_hdatetime.py | 11 +- helpers/test/test_hdocker.py | 19 +- helpers/test/test_henv.py | 3 +- helpers/test/test_hgit.py | 23 +- helpers/test/test_hintrospection.py | 18 +- helpers/test/test_hio.py | 9 +- helpers/test/test_hmarkdown.py | 22 - helpers/test/test_hnumpy.py | 2 +- helpers/test/test_hopenai.py | 46 +- helpers/test/test_hpandas.py | 57 +- helpers/test/test_hparquet.py | 38 +- helpers/test/test_hplayback.py | 4 +- helpers/test/test_hprint.py | 18 +- helpers/test/test_hs3.py | 13 +- helpers/test/test_hsecrets.py | 16 +- helpers/test/test_hserver.py | 1 - helpers/test/test_hstring.py | 3 - helpers/test/test_hunit_test.py | 20 +- helpers/test/test_hunit_test_mock.py | 13 +- helpers/test/test_joblib_helpers.py | 7 +- helpers/test/test_lib_tasks.py | 45 +- helpers/test/test_lib_tasks_docker.py | 27 +- helpers/test/test_lib_tasks_docker_release.py | 8 +- helpers/test/test_lib_tasks_git.py | 12 +- helpers/test/test_lib_tasks_integrate.py | 6 +- helpers/test/test_lib_tasks_pytest.py | 69 +- .../test_master_buildmeister_dashboard.py | 6 +- helpers/test/test_repo_config_amp.py | 1 - helpers/test/test_repo_config_utils.py | 1 - helpers/unit_test_template.py | 1 - import_check/detect_import_cycles.py | 3 +- import_check/show_imports.py | 5 +- .../test/test_detect_import_cycles.py | 48 +- import_check/test/test_show_imports.py | 6 +- linters/action.py | 1 - linters/add_python_init_files.py | 3 +- linters/amp_add_class_frames.py | 10 +- linters/amp_add_toc_to_notebook.py | 4 +- linters/amp_autoflake.py | 3 +- linters/amp_black.py | 8 +- linters/amp_check_file_size.py | 4 +- linters/amp_check_filename.py | 4 +- linters/amp_check_import.py | 5 +- linters/amp_check_md_reference.py | 3 +- linters/amp_check_md_toc_headers.py | 8 +- linters/amp_check_merge_conflict.py | 3 +- linters/amp_check_shebang.py | 3 +- linters/amp_class_method_order.py | 8 +- linters/amp_doc_formatter.py | 14 +- linters/amp_fix_comments.py | 4 +- linters/amp_fix_md_links.py | 4 +- linters/amp_fix_whitespaces.py | 3 +- linters/amp_flake8.py | 4 +- linters/amp_format_separating_line.py | 3 +- linters/amp_isort.py | 4 +- linters/amp_lint_md.py | 19 +- linters/amp_mypy.py | 5 +- linters/amp_normalize_import.py | 22 +- linters/amp_processjupytext.py | 7 +- linters/amp_pylint.py | 5 +- linters/amp_remove_empty_lines_in_function.py | 9 +- .../amp_warn_incorrectly_formatted_todo.py | 3 +- linters/base.py | 7 +- linters/test/test_amp_check_import.py | 2 +- linters/test/test_amp_check_md_reference.py | 5 +- linters/test/test_amp_check_md_toc_headers.py | 2 - linters/test/test_amp_dev_scripts.py | 1 - linters/test/test_amp_doc_formatter.py | 2 - linters/test/test_amp_fix_comment.py | 8 +- linters/test/test_amp_fix_md_links.py | 10 +- linters/test/test_amp_normalize_import.py | 41 +- ...test_amp_remove_empty_lines_in_function.py | 1 - linters/utils.py | 4 +- notebook_template.ipynb | 2 +- notebook_template.py | 2 +- pyproject.toml | 2 +- template_unit_test.py | 1 - 272 files changed, 2160 insertions(+), 1728 deletions(-) diff --git a/config_root/config/config_.py b/config_root/config/config_.py index 8b03b4435..ab6215ea2 100644 --- a/config_root/config/config_.py +++ b/config_root/config/config_.py @@ -593,7 +593,7 @@ def __init__( # TODO(gp): This might be a separate constructor, but it gives problems # with `Config.from_python()`. if array is not None: - #TODO(Juraj): The variable name is not fitting now + # TODO(Juraj): The variable name is not fitting now # In Python 3.12 `from_python` now returns Dict. iterable = array.items() if isinstance(array, dict) else array for key, val in iterable: @@ -690,7 +690,9 @@ def __setitem__( write-after-read (see above) - `None` to use the value set in the constructor """ - _LOG.debug("-> " + hprint.to_str("key val update_mode clobber_mode self")) + _LOG.debug( + "-> " + hprint.to_str("key val update_mode clobber_mode self") + ) clobber_mode = self._resolve_clobber_mode(clobber_mode) report_mode = self._resolve_report_mode(report_mode) try: @@ -1200,13 +1202,15 @@ def _parse_compound_key(key: CompoundKey) -> Tuple[str, Iterable[str]]: tail_compound_key, ) hdbg.dassert_isinstance( - head_scalar_key, ScalarKeyValidTypes, "Keys can only be string or int" + head_scalar_key, + ScalarKeyValidTypes, + "Keys can only be string or int", ) return head_scalar_key, tail_compound_key @staticmethod def _get_config_from_flattened_dict( - flattened_config: Dict[Tuple[str], Any] + flattened_config: Dict[Tuple[str], Any], ) -> "Config": """ Build a config from the flattened config representation. @@ -1394,7 +1398,9 @@ def _get_item( ret = self._config._get_marked_as_used(key) # type: ignore else: # Return the value associated to the key. - ret = self._config.__getitem__(key, mark_key_as_used=mark_key_as_used) # type: ignore + ret = self._config.__getitem__( + key, mark_key_as_used=mark_key_as_used + ) # type: ignore return ret def _resolve_update_mode(self, value: Optional[str]) -> str: diff --git a/config_root/config/config_builder.py b/config_root/config/config_builder.py index e968aa95f..85bf494a7 100644 --- a/config_root/config/config_builder.py +++ b/config_root/config/config_builder.py @@ -134,9 +134,9 @@ def patch_config_list( # Add experiment result dir. dst_subdir = f"result_{idx}" experiment_result_dir = os.path.join(dst_dir, dst_subdir) - config[ - ("backtest_config", "experiment_result_dir") - ] = experiment_result_dir + config[("backtest_config", "experiment_result_dir")] = ( + experiment_result_dir + ) # configs_out.append(config) # Create a copy of the ConfigList and update the internal `configs`. diff --git a/config_root/config/config_list_builder.py b/config_root/config/config_list_builder.py index 592e102ac..0fef46c28 100644 --- a/config_root/config/config_list_builder.py +++ b/config_root/config/config_list_builder.py @@ -272,7 +272,9 @@ def build_config_list_varying_tiled_periods( offset = pd.tseries.frequencies.to_offset(freq_as_pd_str) end_timestamp_tmp += offset _LOG.debug(hprint.to_str("start_timestamp end_timestamp_tmp")) - dates = pd.date_range(start_timestamp, end_timestamp_tmp, freq=freq_as_pd_str) + dates = pd.date_range( + start_timestamp, end_timestamp_tmp, freq=freq_as_pd_str + ) dates = dates.to_list() hdbg.dassert_lte(1, len(dates)) _LOG.debug(hprint.to_str("dates")) @@ -385,7 +387,8 @@ def build_config_list_with_tiled_universe_and_periods( config_list_out.configs = [config] config_list_out = apply_build_config_list(func, config_list_out) _LOG.info( - "After applying universe tiles: num_config_list=%s", len(config_list_out) + "After applying universe tiles: num_config_list=%s", + len(config_list_out), ) hdbg.dassert_lte(1, len(config_list_out)) # Apply the cross-product by the time tiles. diff --git a/config_root/config/test/test_config.py b/config_root/config/test/test_config.py index 88b093fa6..a5b44024c 100644 --- a/config_root/config/test/test_config.py +++ b/config_root/config/test/test_config.py @@ -93,7 +93,6 @@ def _get_flat_config1(self: Any) -> cconfig.Config: class Test_flat_config_set1(hunitest.TestCase): - def test_set1(self) -> None: """ Set a key and print a flat config. @@ -128,7 +127,9 @@ def test_config_with_function(self) -> None: "[(, {'thr': 90})]" ) # Check. - expected_result = "filters: [(, {'thr': 90})]" + expected_result = ( + "filters: [(, {'thr': 90})]" + ) actual_result = str(config) self.assertEqual(actual_result, expected_result) @@ -323,7 +324,9 @@ def test_non_existing_key_with_type1(self) -> None: """ config = _get_flat_config2(self) with self.assertRaises(AssertionError) as cm: - _ = config.get("nrows", "hello", str, report_mode="verbose_exception") + _ = config.get( + "nrows", "hello", str, report_mode="verbose_exception" + ) act = str(cm.exception) exp = """ * Failed assertion * @@ -338,7 +341,6 @@ def test_non_existing_key_with_type1(self) -> None: class Test_flat_config_in1(hunitest.TestCase): - def test_in1(self) -> None: """ Test in operator. @@ -400,7 +402,6 @@ def _get_nested_config1(self: Any) -> cconfig.Config: class Test_nested_config_get1(hunitest.TestCase): - def test_existing_key1(self) -> None: """ Check that a key exists. @@ -498,7 +499,6 @@ def test_non_existing_key3(self) -> None: class Test_nested_config_set1(hunitest.TestCase): - def test_not_existing_key1(self) -> None: """ Set a key that doesn't exist. @@ -743,7 +743,6 @@ def _get_nested_config5(self: Any) -> cconfig.Config: class Test_nested_config_misc1(hunitest.TestCase): - def test_config_print1(self) -> None: """ Test printing a config. @@ -813,7 +812,6 @@ def test_config1(self) -> None: class Test_nested_config_in1(hunitest.TestCase): - def test_in1(self) -> None: """ Test `in` with nested access. @@ -870,7 +868,6 @@ def test_not_in4(self) -> None: class Test_nested_config_update1(hunitest.TestCase): - def test_update1(self) -> None: config1 = _get_nested_config3(self) config2 = _get_nested_config4(self) @@ -1152,7 +1149,6 @@ def _get_nested_config6(self: Any) -> cconfig.Config: class Test_nested_config_flatten1(hunitest.TestCase): - def test_flatten1(self) -> None: # Build config. config = cconfig.Config() @@ -1212,7 +1208,6 @@ def test_flatten2(self) -> None: class Test_subtract_config1(hunitest.TestCase): - def test1(self) -> None: config1 = cconfig.Config() config1[("l0",)] = "1st_floor" @@ -1262,7 +1257,6 @@ def test2(self) -> None: class Test_dassert_is_serializable1(hunitest.TestCase): - def test1(self) -> None: """ Test a config that can be serialized correctly. @@ -1330,7 +1324,6 @@ def test2(self) -> None: class Test_from_env_var1(hunitest.TestCase): - @pytest.mark.requires_ck_infra def test1(self) -> None: eval_config = cconfig.Config.from_dict( @@ -1364,7 +1357,6 @@ def test1(self) -> None: class Test_make_read_only1(hunitest.TestCase): - def test_set1(self) -> None: """ Setting a value that already exists on a read-only config raises. @@ -1496,7 +1488,6 @@ def test_set4(self) -> None: class Test_to_dict1(hunitest.TestCase): - def helper( self, config_as_dict: Dict[str, Any], @@ -1643,7 +1634,6 @@ def test3(self) -> None: class Test_to_dict2(hunitest.TestCase): - def test1(self) -> None: config = _get_nested_config6(self) # Run. @@ -1678,7 +1668,6 @@ def test2(self) -> None: class Test_get_config_from_flattened_dict1(hunitest.TestCase): - def test1(self) -> None: flattened = collections.OrderedDict( [ @@ -1730,7 +1719,6 @@ def test2(self) -> None: class Test_from_dict1(hunitest.TestCase): - def test1(self) -> None: nested = { "read_data": { @@ -1828,7 +1816,6 @@ def test4(self) -> None: class Test_to_pickleable_string(hunitest.TestCase): - def helper( self, value: Any, @@ -1904,7 +1891,6 @@ def test2(self) -> None: class Test_save_to_file(hunitest.TestCase): - def helper(self, value: Optional[str]) -> None: # Set config. log_dir = self.get_scratch_space() @@ -1964,7 +1950,6 @@ def remove_line_numbers(actual_config: str) -> str: class Test_to_string(hunitest.TestCase): - def get_test_config( self, value: Any, @@ -1981,7 +1966,9 @@ def test1(self) -> None: """ Test when a value is a DataFrame. """ - value = pd.DataFrame(data=[[1, 2, 3], [4, 5, 6]], columns=["a", "b", "c"]) + value = pd.DataFrame( + data=[[1, 2, 3], [4, 5, 6]], columns=["a", "b", "c"] + ) config = self.get_test_config(value) # mode = "verbose" @@ -2104,7 +2091,6 @@ def test6(self) -> None: class Test_mark_as_used1(hunitest.TestCase): - def test1(self) -> None: """ Test marking a config with scalar values. @@ -2134,7 +2120,10 @@ def test2(self) -> None: expected_value = r"key3: value3" actual_value = test_nested_config.get_and_mark_as_used("key2") self.assert_equal( - str(actual_value), expected_value, purify_text=True, fuzzy_match=True + str(actual_value), + expected_value, + purify_text=True, + fuzzy_match=True, ) # Test marking the subconfig. expected_config = r""" @@ -2156,7 +2145,10 @@ def test3(self) -> None: key4: value3""" actual_value = test_nested_config.get_and_mark_as_used("key2") self.assert_equal( - str(actual_value), expected_value, purify_text=True, fuzzy_match=True + str(actual_value), + expected_value, + purify_text=True, + fuzzy_match=True, ) # expected_config = r""" @@ -2176,7 +2168,10 @@ def test4(self) -> None: expected_value = "['value2', 2]" actual_value = test_config.get_and_mark_as_used("key2") self.assert_equal( - str(actual_value), expected_value, purify_text=True, fuzzy_match=True + str(actual_value), + expected_value, + purify_text=True, + fuzzy_match=True, ) # expected_config = r""" @@ -2233,7 +2228,6 @@ def test1(self) -> None: class Test_check_unused_variables1(hunitest.TestCase): - def test1(self) -> None: """ Verify that a single unused variable is correctly identified. @@ -2270,7 +2264,11 @@ def test3(self) -> None: } config = cconfig.Config.from_dict(config) unused_variables = config.check_unused_variables() - expected = [("key1",), ("key2", "key3", "key4"), ("key2", "key3", "key5")] + expected = [ + ("key1",), + ("key2", "key3", "key4"), + ("key2", "key3", "key5"), + ] self.assertListEqual(unused_variables, expected) @@ -2319,7 +2317,10 @@ def raise_stmt( self.assert_equal(act, exp, purify_text=True, fuzzy_match=True) def run_steps_assert_string( - self, workload: List[Tuple[str, Optional[str]]], mode: str, globals_: Dict + self, + workload: List[Tuple[str, Optional[str]]], + mode: str, + globals_: Dict, ) -> None: for data in workload: hdbg.dassert_eq(len(data), 2, "Invalid data='%s'", str(data)) @@ -2424,7 +2425,6 @@ def test_check_string_repr1(self) -> None: class Test_basic1(_Config_execute_stmt_TestCase1): - def test1(self) -> None: """ Various assignments and their representations. diff --git a/config_root/config/test/test_config_builder.py b/config_root/config/test/test_config_builder.py index 493bd9c56..09f47f83f 100644 --- a/config_root/config/test/test_config_builder.py +++ b/config_root/config/test/test_config_builder.py @@ -38,9 +38,7 @@ def test1(self) -> None: """ Build a config from. """ - config_builder = ( - "config_root.config.test.test_config_builder._build_test_config_list()" - ) + config_builder = "config_root.config.test.test_config_builder._build_test_config_list()" configs = cconfig.get_config_list_from_builder(config_builder) txt = str(configs) self.check_string(txt, purify_text=True) diff --git a/config_root/config/test/test_config_list_builder.py b/config_root/config/test/test_config_list_builder.py index 4152372e0..e5497026f 100644 --- a/config_root/config/test/test_config_list_builder.py +++ b/config_root/config/test/test_config_list_builder.py @@ -176,9 +176,9 @@ def test1(self) -> None: # Prepare inputs. # Create a Config with `overwrite` mode to support item reassignment. system_config = cconfig.Config(update_mode="overwrite") - system_config[ - "backtest_config", "time_interval_str" - ] = "2020-01-01_2020-03-01" + system_config["backtest_config", "time_interval_str"] = ( + "2020-01-01_2020-03-01" + ) system_config["backtest_config", "freq_as_pd_str"] = "M" system_config["backtest_config", "lookback_as_pd_str"] = "90D" system_config["market_data_config", "asset_ids"] = [13684, 10971] diff --git a/config_root/config/test/test_config_utils.py b/config_root/config/test/test_config_utils.py index 7e41bedce..2832e317d 100644 --- a/config_root/config/test/test_config_utils.py +++ b/config_root/config/test/test_config_utils.py @@ -79,7 +79,6 @@ def _get_test_config4() -> cconfig.Config: # TODO(gp): -> validate_config_list class Test_validate_configs1(hunitest.TestCase): - def test_check_same_configs_error(self) -> None: """ Verify that an error is raised when duplicated configs are encountered. @@ -117,7 +116,6 @@ def test1(self) -> None: class Test_apply_config_overrides_from_command_line1(hunitest.TestCase): - def test1(self) -> None: """ Verify that config values are updated correctly. @@ -166,7 +164,6 @@ def test2(self) -> None: class Test_intersect_configs1(hunitest.TestCase): - def test_same_config(self) -> None: """ Verify that intersection of two same configs equals those configs. @@ -204,7 +201,6 @@ def test1(self) -> None: class Test_subtract_configs1(hunitest.TestCase): - def test_same_config(self) -> None: """ Verify that the difference of two configs is empty. @@ -294,7 +290,6 @@ def test3(self) -> None: class Test_diff_configs1(hunitest.TestCase): - def test_same_config(self) -> None: """ Verify that the difference of two configs is empty. @@ -338,7 +333,9 @@ def test1(self) -> None: {"build_targets": {"target_asset": "Crude Oil"}} ), # - cconfig.Config.from_dict({"build_targets": {"target_asset": "Gold"}}), + cconfig.Config.from_dict( + {"build_targets": {"target_asset": "Gold"}} + ), ] self.assert_equal(str(act), str(exp)) @@ -356,10 +353,15 @@ def test2(self) -> None: {"build_targets": {"target_asset": "Crude Oil"}} ), # - cconfig.Config.from_dict({"build_targets": {"target_asset": "Gold"}}), + cconfig.Config.from_dict( + {"build_targets": {"target_asset": "Gold"}} + ), # cconfig.Config.from_dict( - {"build_targets": {"target_asset": "Crude Oil"}, "hello": "world"} + { + "build_targets": {"target_asset": "Crude Oil"}, + "hello": "world", + } ), ] exp = "\n".join(map(str, exp)) @@ -372,7 +374,6 @@ def test2(self) -> None: class Test_convert_to_dataframe1(hunitest.TestCase): - def test1(self) -> None: """ Compute and verify dataframe with all config parameters. @@ -404,7 +405,6 @@ def test1(self) -> None: class Test_build_config_diff_dataframe1(hunitest.TestCase): - def test1(self) -> None: """ Summarize differences between two different configs. @@ -467,7 +467,6 @@ def test3(self) -> None: class Test_make_hashable(hunitest.TestCase): - def helper(self, obj: Any, is_hashable: bool, expected: str) -> None: is_hashable_before = isinstance(obj, collections.abc.Hashable) self.assertEqual(is_hashable_before, is_hashable) @@ -569,7 +568,6 @@ def test6(self) -> None: class Test_replace_shared_root_path(hunitest.TestCase): - def test_replace_shared_dir_paths(self) -> None: """ Test replacing in config all shared root paths with the dummy mapping. @@ -621,7 +619,6 @@ def test_replace_shared_dir_paths(self) -> None: class Test_load_config_from_pickle1(hunitest.TestCase): - def helper( self, expected_config_version: str, expected_signature: str ) -> None: diff --git a/dev_scripts_helpers/aws/am_aws.py b/dev_scripts_helpers/aws/am_aws.py index 1f3147df3..23b0644d4 100755 --- a/dev_scripts_helpers/aws/am_aws.py +++ b/dev_scripts_helpers/aws/am_aws.py @@ -87,7 +87,8 @@ def _main(parser: argparse.ArgumentParser) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--action", diff --git a/dev_scripts_helpers/chatgpt/manage_chatgpt_assistant.py b/dev_scripts_helpers/chatgpt/manage_chatgpt_assistant.py index 3ffd87da1..34f97528b 100755 --- a/dev_scripts_helpers/chatgpt/manage_chatgpt_assistant.py +++ b/dev_scripts_helpers/chatgpt/manage_chatgpt_assistant.py @@ -113,7 +113,9 @@ def _main(parser: argparse.ArgumentParser) -> None: name=args.new_name, tools=tools, model=args.model, - file_ids=[hchatgp.get_gpt_id(path) for path in args.input_file_paths], + file_ids=[ + hchatgp.get_gpt_id(path) for path in args.input_file_paths + ], ) elif args.delete_name: assistant_id = hchatgp.get_assistant_id_by_name(args.delete_name) diff --git a/dev_scripts_helpers/chatgpt/run_simple_chatgpt.py b/dev_scripts_helpers/chatgpt/run_simple_chatgpt.py index 09439253a..8254bed7b 100755 --- a/dev_scripts_helpers/chatgpt/run_simple_chatgpt.py +++ b/dev_scripts_helpers/chatgpt/run_simple_chatgpt.py @@ -44,7 +44,8 @@ def _process_text(txt: str, instruction: str) -> str: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--instruction", required=True) hparser.add_input_output_args(parser) diff --git a/dev_scripts_helpers/cleanup_scripts/CmTask1292_Rename_old_aws_env_vars.py b/dev_scripts_helpers/cleanup_scripts/CmTask1292_Rename_old_aws_env_vars.py index 68d35096c..d7dcfdce0 100755 --- a/dev_scripts_helpers/cleanup_scripts/CmTask1292_Rename_old_aws_env_vars.py +++ b/dev_scripts_helpers/cleanup_scripts/CmTask1292_Rename_old_aws_env_vars.py @@ -21,7 +21,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_verbosity_arg(parser) return parser diff --git a/dev_scripts_helpers/cleanup_scripts/CmTask972_Merge_dataframe_to_str_and_df_to_short_str_into_hpandas.py b/dev_scripts_helpers/cleanup_scripts/CmTask972_Merge_dataframe_to_str_and_df_to_short_str_into_hpandas.py index 2d8b3ba75..7c987b477 100755 --- a/dev_scripts_helpers/cleanup_scripts/CmTask972_Merge_dataframe_to_str_and_df_to_short_str_into_hpandas.py +++ b/dev_scripts_helpers/cleanup_scripts/CmTask972_Merge_dataframe_to_str_and_df_to_short_str_into_hpandas.py @@ -109,7 +109,8 @@ def _run() -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_verbosity_arg(parser) return parser diff --git a/dev_scripts_helpers/coding_tools/diff_to_vimdiff.py b/dev_scripts_helpers/coding_tools/diff_to_vimdiff.py index 519116944..1e03032f1 100755 --- a/dev_scripts_helpers/coding_tools/diff_to_vimdiff.py +++ b/dev_scripts_helpers/coding_tools/diff_to_vimdiff.py @@ -189,9 +189,7 @@ def _parse_diff_output( Only in /Users/saggese/src/amp1/dataflow_amp/real_time/test: TestReal... ``` """ - print( - hprint.frame(f"Compare file content in dirs '{dir1}' vs '{dir2}'") - ) + print(hprint.frame(f"Compare file content in dirs '{dir1}' vs '{dir2}'")) # Read the output from `diff -r --brief`. hdbg.dassert_path_exists(input_file) _LOG.info("Reading '%s'", input_file) @@ -322,7 +320,8 @@ def _parse_diff_output( def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) # Specify dirs. parser.add_argument( diff --git a/dev_scripts_helpers/coding_tools/find_unused_golden_files.py b/dev_scripts_helpers/coding_tools/find_unused_golden_files.py index 63146a9d8..2a2f90a0b 100755 --- a/dev_scripts_helpers/coding_tools/find_unused_golden_files.py +++ b/dev_scripts_helpers/coding_tools/find_unused_golden_files.py @@ -101,7 +101,9 @@ def parse_test_code( if ( "self.check_string(" in test_method_code or "self.check_dataframe(" in test_method_code - or any(f"self.{n}(" in test_method_code for n in helper_names) + or any( + f"self.{n}(" in test_method_code for n in helper_names + ) ) and not test_method_name.startswith("test"): helper_names.add(test_method_name) for test_method_name, test_method_code in methods_grouped: diff --git a/dev_scripts_helpers/coding_tools/grsync.py b/dev_scripts_helpers/coding_tools/grsync.py index e1429b41b..98374733d 100755 --- a/dev_scripts_helpers/coding_tools/grsync.py +++ b/dev_scripts_helpers/coding_tools/grsync.py @@ -154,13 +154,20 @@ def _get_list_files_cmd( def _main(): parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( - "--src_dir", action="store", required=True, help="Local directory to sync" + "--src_dir", + action="store", + required=True, + help="Local directory to sync", ) parser.add_argument( - "--dst_dir", action="store", default=None, help="Remote directory to sync" + "--dst_dir", + action="store", + default=None, + help="Remote directory to sync", ) parser.add_argument( "--action", diff --git a/dev_scripts_helpers/coding_tools/invite_github_collaborator.py b/dev_scripts_helpers/coding_tools/invite_github_collaborator.py index 00a1a5046..306cde091 100644 --- a/dev_scripts_helpers/coding_tools/invite_github_collaborator.py +++ b/dev_scripts_helpers/coding_tools/invite_github_collaborator.py @@ -83,7 +83,9 @@ def _invite_collaborator( repo_name, "invitations", ) - response = requests.get(invitation_check_url, headers=headers, timeout=10) + response = requests.get( + invitation_check_url, headers=headers, timeout=10 + ) status_code = response.status_code if status_code == 200: # Check if an invitation was sent to a user that is already a GH collaborator. diff --git a/dev_scripts_helpers/coding_tools/manage_cache.py b/dev_scripts_helpers/coding_tools/manage_cache.py index 7d873a9ab..926b6e2de 100755 --- a/dev_scripts_helpers/coding_tools/manage_cache.py +++ b/dev_scripts_helpers/coding_tools/manage_cache.py @@ -47,7 +47,8 @@ def _test2() -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--action", required=True, type=str) hparser.add_verbosity_arg(parser) diff --git a/dev_scripts_helpers/coding_tools/measure_import_times.py b/dev_scripts_helpers/coding_tools/measure_import_times.py index fcd147c61..8f41143e3 100755 --- a/dev_scripts_helpers/coding_tools/measure_import_times.py +++ b/dev_scripts_helpers/coding_tools/measure_import_times.py @@ -72,7 +72,9 @@ def measure_time(self, module: str) -> Union[str, float]: # Execute python "import module" to measure. if module not in self.checked_modules: timer = Timer() - rc, output = hsystem.system_to_string(f'python -c "import {module}"', abort_on_error=False) + rc, output = hsystem.system_to_string( + f'python -c "import {module}"', abort_on_error=False + ) timer.stop() if rc == 0: # Save execution time. @@ -168,7 +170,8 @@ def _sort_by_time(self) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "-d", diff --git a/dev_scripts_helpers/coding_tools/parallel_script_template.py b/dev_scripts_helpers/coding_tools/parallel_script_template.py index b85e29c43..05a6ffaa7 100755 --- a/dev_scripts_helpers/coding_tools/parallel_script_template.py +++ b/dev_scripts_helpers/coding_tools/parallel_script_template.py @@ -41,7 +41,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--workload", diff --git a/dev_scripts_helpers/coding_tools/process_prof.py b/dev_scripts_helpers/coding_tools/process_prof.py index 29f5b2932..aad8620ee 100755 --- a/dev_scripts_helpers/coding_tools/process_prof.py +++ b/dev_scripts_helpers/coding_tools/process_prof.py @@ -35,7 +35,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_verbosity_arg(parser) parser.add_argument( diff --git a/dev_scripts_helpers/coding_tools/script_template.py b/dev_scripts_helpers/coding_tools/script_template.py index e9b684bf0..00f859083 100644 --- a/dev_scripts_helpers/coding_tools/script_template.py +++ b/dev_scripts_helpers/coding_tools/script_template.py @@ -26,7 +26,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("positional", nargs="*", help="...") parser.add_argument("--dst_dir", action="store", help="Destination dir") diff --git a/dev_scripts_helpers/coding_tools/toml_merge.py b/dev_scripts_helpers/coding_tools/toml_merge.py index 440f2adde..91768d664 100755 --- a/dev_scripts_helpers/coding_tools/toml_merge.py +++ b/dev_scripts_helpers/coding_tools/toml_merge.py @@ -77,7 +77,8 @@ def _merge_toml(pyprojs: List[_DepDict]) -> _DepDict: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--in_file", action="append", help="Files to read", required=True diff --git a/dev_scripts_helpers/coding_tools/traceback_to_cfile.py b/dev_scripts_helpers/coding_tools/traceback_to_cfile.py index 45029759d..e25156cb4 100755 --- a/dev_scripts_helpers/coding_tools/traceback_to_cfile.py +++ b/dev_scripts_helpers/coding_tools/traceback_to_cfile.py @@ -37,7 +37,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) in_default = _NEWEST_LOG_FILE parser = hparser.add_input_output_args( diff --git a/dev_scripts_helpers/coding_tools/transform_template.py b/dev_scripts_helpers/coding_tools/transform_template.py index 93d758e29..f02bbdd68 100755 --- a/dev_scripts_helpers/coding_tools/transform_template.py +++ b/dev_scripts_helpers/coding_tools/transform_template.py @@ -24,7 +24,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_input_output_args(parser) hparser.add_verbosity_arg(parser) diff --git a/dev_scripts_helpers/coding_tools/url.py b/dev_scripts_helpers/coding_tools/url.py index 6c537e632..9e85300b0 100755 --- a/dev_scripts_helpers/coding_tools/url.py +++ b/dev_scripts_helpers/coding_tools/url.py @@ -45,7 +45,8 @@ def _print(tag: str, val: str, verbose: bool) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("positional", nargs="*") parser.add_argument("--short", action="store_true", help="Short output form") diff --git a/dev_scripts_helpers/dockerize/dockerized_template.py b/dev_scripts_helpers/dockerize/dockerized_template.py index e4c06980f..4a05b7831 100755 --- a/dev_scripts_helpers/dockerize/dockerized_template.py +++ b/dev_scripts_helpers/dockerize/dockerized_template.py @@ -19,7 +19,8 @@ def _parse() -> argparse.ArgumentParser: # Create an ArgumentParser instance with the provided docstring. parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) # FILL THIS. # parser.add_argument( diff --git a/dev_scripts_helpers/documentation/convert_docx_to_markdown.py b/dev_scripts_helpers/documentation/convert_docx_to_markdown.py index fa45e55b4..7259ec2cf 100755 --- a/dev_scripts_helpers/documentation/convert_docx_to_markdown.py +++ b/dev_scripts_helpers/documentation/convert_docx_to_markdown.py @@ -107,7 +107,8 @@ def _clean_up_artifacts(md_file: str, md_file_figs: str) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--docx_file", diff --git a/dev_scripts_helpers/documentation/dockerized_graphviz.py b/dev_scripts_helpers/documentation/dockerized_graphviz.py index 68a63e5a4..c8d4831de 100755 --- a/dev_scripts_helpers/documentation/dockerized_graphviz.py +++ b/dev_scripts_helpers/documentation/dockerized_graphviz.py @@ -18,7 +18,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("-i", "--input", action="store", required=True) parser.add_argument("-o", "--output", action="store", required=True) diff --git a/dev_scripts_helpers/documentation/dockerized_latex.py b/dev_scripts_helpers/documentation/dockerized_latex.py index 8c28fe0e6..b9cbcb65c 100755 --- a/dev_scripts_helpers/documentation/dockerized_latex.py +++ b/dev_scripts_helpers/documentation/dockerized_latex.py @@ -21,7 +21,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("-i", "--input", action="store", required=True) parser.add_argument("-o", "--output", action="store", required=True) diff --git a/dev_scripts_helpers/documentation/dockerized_mermaid.py b/dev_scripts_helpers/documentation/dockerized_mermaid.py index 45e9c3941..50545f731 100755 --- a/dev_scripts_helpers/documentation/dockerized_mermaid.py +++ b/dev_scripts_helpers/documentation/dockerized_mermaid.py @@ -18,7 +18,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("-i", "--input", action="store", required=True) parser.add_argument("-o", "--output", action="store", default="") diff --git a/dev_scripts_helpers/documentation/dockerized_pandoc.py b/dev_scripts_helpers/documentation/dockerized_pandoc.py index 65a0b2df7..e02e306f0 100755 --- a/dev_scripts_helpers/documentation/dockerized_pandoc.py +++ b/dev_scripts_helpers/documentation/dockerized_pandoc.py @@ -26,13 +26,16 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_dockerized_script_arg(parser) parser.add_argument("--input", action="store") parser.add_argument("--output", action="store", default="") parser.add_argument("--data_dir", action="store") - parser.add_argument("--container_type", action="store", default="pandoc_only") + parser.add_argument( + "--container_type", action="store", default="pandoc_only" + ) hparser.add_verbosity_arg(parser) return parser diff --git a/dev_scripts_helpers/documentation/dockerized_prettier.py b/dev_scripts_helpers/documentation/dockerized_prettier.py index 65e82f5ac..d3f9bcdc8 100755 --- a/dev_scripts_helpers/documentation/dockerized_prettier.py +++ b/dev_scripts_helpers/documentation/dockerized_prettier.py @@ -42,7 +42,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_input_output_args(parser) hparser.add_dockerized_script_arg(parser) diff --git a/dev_scripts_helpers/documentation/dockerized_tikz_to_bitmap.py b/dev_scripts_helpers/documentation/dockerized_tikz_to_bitmap.py index 4440dd854..7da316469 100755 --- a/dev_scripts_helpers/documentation/dockerized_tikz_to_bitmap.py +++ b/dev_scripts_helpers/documentation/dockerized_tikz_to_bitmap.py @@ -19,7 +19,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("-i", "--input", action="store", required=True) parser.add_argument("-o", "--output", action="store", required=True) diff --git a/dev_scripts_helpers/documentation/generate_script_catalog.py b/dev_scripts_helpers/documentation/generate_script_catalog.py index f79a3049a..d0940ede6 100755 --- a/dev_scripts_helpers/documentation/generate_script_catalog.py +++ b/dev_scripts_helpers/documentation/generate_script_catalog.py @@ -46,7 +46,8 @@ def _get_docstring(file_name: str) -> str: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--src_dir", action="store", default=".") parser.add_argument("--src_file", action="store", default=None) @@ -89,7 +90,10 @@ def _main(parser: argparse.ArgumentParser) -> None: ordered_file_names = sorted(res.keys(), key=lambda x: len(x.split("/"))) for file_name in ordered_file_names: _LOG.debug( - "file_name=%s curr_dir=%s last_dir=%s", file_name, curr_dir, last_dir + "file_name=%s curr_dir=%s last_dir=%s", + file_name, + curr_dir, + last_dir, ) docstring = res[file_name] file_name = file_name.replace("./", "") diff --git a/dev_scripts_helpers/documentation/lint_notes.py b/dev_scripts_helpers/documentation/lint_notes.py index 781c1b02a..8e03b3ff3 100755 --- a/dev_scripts_helpers/documentation/lint_notes.py +++ b/dev_scripts_helpers/documentation/lint_notes.py @@ -271,8 +271,7 @@ def _frame_chapters(txt: str, *, max_lev: int = 4) -> str: txt_processed = True else: _LOG.debug( - " -> Skip formatting the chapter frame: lev=%d, " - "max_lev=%d", + " -> Skip formatting the chapter frame: lev=%d, max_lev=%d", lev, max_lev, ) @@ -415,7 +414,8 @@ def _process( def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_input_output_args(parser) parser.add_argument( diff --git a/dev_scripts_helpers/documentation/notes_to_pdf.py b/dev_scripts_helpers/documentation/notes_to_pdf.py index 7899f8a0f..0df86a3d8 100755 --- a/dev_scripts_helpers/documentation/notes_to_pdf.py +++ b/dev_scripts_helpers/documentation/notes_to_pdf.py @@ -18,7 +18,6 @@ --no_cleanup --no_cleanup_before --no_run_latex_again --no_open """ - import argparse import logging import os @@ -64,7 +63,9 @@ def _log_system(cmd: str) -> None: def _system(cmd: str, *, log_level: int = logging.DEBUG, **kwargs: Any) -> int: _log_system(cmd) - rc = hsystem.system(cmd, log_level=log_level, suppress_output=False, **kwargs) + rc = hsystem.system( + cmd, log_level=log_level, suppress_output=False, **kwargs + ) return rc # type: ignore @@ -318,7 +319,9 @@ def _run_pandoc_to_pdf( ) _LOG.debug("%s", "before: " + hprint.to_str("cmd")) if not use_host_tools: - cmd = hdocker.run_dockerized_latex(cmd, mode="return_cmd", use_sudo=False) + cmd = hdocker.run_dockerized_latex( + cmd, mode="return_cmd", use_sudo=False + ) _LOG.debug("%s", "after: " + hprint.to_str("cmd")) _ = _system(cmd) # - Run latex again. @@ -535,7 +538,9 @@ def _run_all(args: argparse.Namespace) -> None: # Print actions. actions = hparser.select_actions(args, _VALID_ACTIONS, _DEFAULT_ACTIONS) add_frame = True - actions_as_str = hparser.actions_to_string(actions, _VALID_ACTIONS, add_frame) + actions_as_str = hparser.actions_to_string( + actions, _VALID_ACTIONS, add_frame + ) _LOG.info("\n%s", actions_as_str) if args.preview_actions: return @@ -570,7 +575,9 @@ def _run_all(args: argparse.Namespace) -> None: action = "preprocess_notes" to_execute, actions = _mark_action(action, actions) if to_execute: - file_name = _preprocess_notes(file_name, prefix, args.type, args.toc_type) + file_name = _preprocess_notes( + file_name, prefix, args.type, args.toc_type + ) # - Render_images action = "render_images" to_execute, actions = _mark_action(action, actions) @@ -664,7 +671,8 @@ def _run_all(args: argparse.Namespace) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("-i", "--input", action="store", type=str, required=True) parser.add_argument( diff --git a/dev_scripts_helpers/documentation/preprocess_notes.py b/dev_scripts_helpers/documentation/preprocess_notes.py index f77258a3f..bee0f7783 100755 --- a/dev_scripts_helpers/documentation/preprocess_notes.py +++ b/dev_scripts_helpers/documentation/preprocess_notes.py @@ -146,7 +146,9 @@ def _process_question_to_markdown(line: str) -> Tuple[bool, str]: return do_continue, line -def _process_question_to_slides(line: str, *, level: int = 4) -> Tuple[bool, str]: +def _process_question_to_slides( + line: str, *, level: int = 4 +) -> Tuple[bool, str]: """ Transform `* foo bar` into `#### foo bar`. """ @@ -261,9 +263,9 @@ def _transform_lines(txt: str, type_: str, *, is_qa: bool = False) -> str: out.append(" " * _NUM_SPACES + line) else: # Empty line. - prev_line_is_verbatim = ((i - 1) > 0) and lines[i - 1].startswith( - "```" - ) + prev_line_is_verbatim = ((i - 1) > 0) and lines[ + i - 1 + ].startswith("```") next_line_is_verbatim = ((i + 1) < len(lines)) and ( lines[i + 1].startswith("```") ) @@ -353,7 +355,8 @@ def _add_navigation_slides( def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--input", action="store", type=str, required=True) parser.add_argument("--output", action="store", type=str, default=None) diff --git a/dev_scripts_helpers/documentation/publish_notes.py b/dev_scripts_helpers/documentation/publish_notes.py index eadab340a..b2e252129 100755 --- a/dev_scripts_helpers/documentation/publish_notes.py +++ b/dev_scripts_helpers/documentation/publish_notes.py @@ -116,7 +116,8 @@ def _publish_all_files(args: argparse.Namespace) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("positional", nargs="*", choices=["rm", "ls", "publish"]) parser.add_argument("--no_incremental", action="store_true") diff --git a/dev_scripts_helpers/documentation/render_images.py b/dev_scripts_helpers/documentation/render_images.py index 10a3d8034..6a4b2e797 100755 --- a/dev_scripts_helpers/documentation/render_images.py +++ b/dev_scripts_helpers/documentation/render_images.py @@ -367,7 +367,9 @@ def _get_comment_prefix_postfix(extension: str) -> Tuple[str, str]: return comment_prefix, comment_postfix -def _insert_image_code(extension: str, rel_img_path: str, user_img_size: str) -> str: +def _insert_image_code( + extension: str, rel_img_path: str, user_img_size: str +) -> str: """ Insert the code to display the image in the output file. """ @@ -489,7 +491,8 @@ def _render_images( # E.g., "plantuml" or "mermaid". image_code_type = m.group(2) hdbg.dassert_in( - image_code_type, ["plantuml", "mermaid", "tikz", "graphviz", "latex"] + image_code_type, + ["plantuml", "mermaid", "tikz", "graphviz", "latex"], ) if m.group(3): hdbg.dassert_eq(user_rel_img_path, "") @@ -532,7 +535,9 @@ def _render_images( state, line, comment_prefix, comment_postfix ) ) - out_lines.append(_insert_image_code(extension, rel_img_path, user_img_size)) + out_lines.append( + _insert_image_code(extension, rel_img_path, user_img_size) + ) user_img_size = "" # Set the parser to search for a new image code block. if state == "found_image_code": @@ -594,7 +599,8 @@ def _open_html(out_file: str) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) # Add input and output file arguments. parser.add_argument( diff --git a/dev_scripts_helpers/documentation/replace_latex.py b/dev_scripts_helpers/documentation/replace_latex.py index 72ff2c455..2d6ce81bc 100755 --- a/dev_scripts_helpers/documentation/replace_latex.py +++ b/dev_scripts_helpers/documentation/replace_latex.py @@ -91,7 +91,8 @@ def _repl_func(m: re.Match) -> str: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "-a", @@ -118,12 +119,16 @@ def _main(parser: argparse.ArgumentParser) -> None: cmd = f"git checkout -- {args.file}" _ = hsystem.system(cmd) if "pandoc_before" in actions: - cmd = f"notes_to_pdf.py -a pdf --no_toc --no_open_pdf --input {args.file}" + cmd = ( + f"notes_to_pdf.py -a pdf --no_toc --no_open_pdf --input {args.file}" + ) _ = hsystem.system(cmd) if "replace" in actions: _standard_cleanup(args.file, args.aggressive) if "pandoc_after" in actions: - cmd = f"notes_to_pdf.py -a pdf --no_toc --no_open_pdf --input {args.file}" + cmd = ( + f"notes_to_pdf.py -a pdf --no_toc --no_open_pdf --input {args.file}" + ) _ = hsystem.system(cmd) diff --git a/dev_scripts_helpers/documentation/run_pandoc.py b/dev_scripts_helpers/documentation/run_pandoc.py index af5adb84b..fd7f6080e 100755 --- a/dev_scripts_helpers/documentation/run_pandoc.py +++ b/dev_scripts_helpers/documentation/run_pandoc.py @@ -31,7 +31,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_input_output_args(parser) parser.add_argument( diff --git a/dev_scripts_helpers/documentation/test/test_dockerized_pandoc.py b/dev_scripts_helpers/documentation/test/test_dockerized_pandoc.py index c6cbc84f8..738bed986 100644 --- a/dev_scripts_helpers/documentation/test/test_dockerized_pandoc.py +++ b/dev_scripts_helpers/documentation/test/test_dockerized_pandoc.py @@ -15,7 +15,6 @@ class Test_Pandoc_Cmd_Conversion(hunitest.TestCase): - def test1(self) -> None: """ Test `convert_pandoc_cmd_to_arguments` to parse a pandoc command string @@ -71,7 +70,6 @@ def test2(self) -> None: class Test_run_dockerized_pandoc(hunitest.TestCase): - @pytest.mark.skipif( hserver.is_inside_ci() or hserver.is_dev_csfy(), reason="Disabled because of CmampTask10710", diff --git a/dev_scripts_helpers/documentation/test/test_dockerized_prettier.py b/dev_scripts_helpers/documentation/test/test_dockerized_prettier.py index 5302d5170..6fdebf927 100644 --- a/dev_scripts_helpers/documentation/test/test_dockerized_prettier.py +++ b/dev_scripts_helpers/documentation/test/test_dockerized_prettier.py @@ -14,7 +14,6 @@ class Test_run_dockerized_prettier(hunitest.TestCase): - @pytest.mark.skipif( hserver.is_inside_ci() or hserver.is_dev_csfy(), reason="Disabled because of CmampTask10710", diff --git a/dev_scripts_helpers/documentation/test/test_extract_headers_from_markdown.py b/dev_scripts_helpers/documentation/test/test_extract_headers_from_markdown.py index 4a369d2fc..e965d233e 100644 --- a/dev_scripts_helpers/documentation/test/test_extract_headers_from_markdown.py +++ b/dev_scripts_helpers/documentation/test/test_extract_headers_from_markdown.py @@ -14,7 +14,6 @@ class Test_extract_headers_from_markdown1(hunitest.TestCase): - def test1(self) -> None: # Prepare inputs. content = r""" diff --git a/dev_scripts_helpers/documentation/test/test_generate_readme_index.py b/dev_scripts_helpers/documentation/test/test_generate_readme_index.py index 1789da748..60ec11b5b 100644 --- a/dev_scripts_helpers/documentation/test/test_generate_readme_index.py +++ b/dev_scripts_helpers/documentation/test/test_generate_readme_index.py @@ -3,9 +3,7 @@ import pytest -pytest.importorskip( - "openai" -) # noqa: E402 # pylint: disable=wrong-import-position +pytest.importorskip("openai") # noqa: E402 # pylint: disable=wrong-import-position import dev_scripts_helpers.documentation.generate_readme_index as dshdgrein import helpers.hio as hio @@ -18,7 +16,6 @@ class Test_list_markdown_files(hunitest.TestCase): - def test1(self) -> None: """ Test retrieving all Markdown files in a directory. @@ -129,7 +126,6 @@ def _write_input_file(self, txt: str, file_name: str) -> str: class Test_generate_readme_index(hunitest.TestCase): - def test1(self) -> None: """ Test generating README from scratch using placeholder summary. diff --git a/dev_scripts_helpers/documentation/test/test_lint_notes.py b/dev_scripts_helpers/documentation/test/test_lint_notes.py index 1ddf8cd05..3495d6700 100644 --- a/dev_scripts_helpers/documentation/test/test_lint_notes.py +++ b/dev_scripts_helpers/documentation/test/test_lint_notes.py @@ -54,7 +54,6 @@ def _get_text1() -> str: reason="Disabled because of CmampTask10710", ) class Test_lint_notes1(hunitest.TestCase): - def test_preprocess1(self) -> None: txt = r"""$$E_{in} = \frac{1}{N} \sum_i e(h(\vx_i), y_i)$$""" exp = r""" @@ -140,7 +139,6 @@ def _helper_preprocess(self, txt: str, exp: str) -> None: reason="Disabled because of CmampTask10710", ) class Test_lint_notes2(hunitest.TestCase): - def test_process1(self) -> None: txt = _get_text1() exp = None @@ -353,7 +351,6 @@ def _helper_process( reason="Disabled because of CmampTask10710", ) class Test_lint_notes_cmd_line1(hunitest.TestCase): - def create_md_input_file(self) -> str: txt = """ # Header1 diff --git a/dev_scripts_helpers/documentation/test/test_notes_to_pdf.py b/dev_scripts_helpers/documentation/test/test_notes_to_pdf.py index ea3310689..402a372d5 100644 --- a/dev_scripts_helpers/documentation/test/test_notes_to_pdf.py +++ b/dev_scripts_helpers/documentation/test/test_notes_to_pdf.py @@ -25,7 +25,6 @@ reason="Disabled because of CmampTask10710", ) class Test_notes_to_pdf1(hunitest.TestCase): - def create_input_file(self) -> str: txt = """ # Header1 diff --git a/dev_scripts_helpers/documentation/test/test_preprocess_notes.py b/dev_scripts_helpers/documentation/test/test_preprocess_notes.py index b7f91980b..3874950e9 100644 --- a/dev_scripts_helpers/documentation/test/test_preprocess_notes.py +++ b/dev_scripts_helpers/documentation/test/test_preprocess_notes.py @@ -44,7 +44,6 @@ def _run_preprocess_notes(in_file: str, out_file: str) -> str: class Test_process_color_commands1(hunitest.TestCase): - def test_text_content1(self) -> None: """ Test with plain text content. diff --git a/dev_scripts_helpers/documentation/test/test_render_images.py b/dev_scripts_helpers/documentation/test/test_render_images.py index c9254f317..708a26272 100644 --- a/dev_scripts_helpers/documentation/test/test_render_images.py +++ b/dev_scripts_helpers/documentation/test/test_render_images.py @@ -20,7 +20,6 @@ class Test_get_rendered_file_paths1(hunitest.TestCase): - def test1(self) -> None: """ Check generation of file paths for rendering images. @@ -49,7 +48,6 @@ def test1(self) -> None: class Test_ImageHashCache1(hunitest.TestCase): - def test1(self) -> None: """ Test basic functionality of ImageHashCache. @@ -164,7 +162,6 @@ def test1(self) -> None: reason="Disabled because of CmampTask10710", ) class Test_render_image_code1(hunitest.TestCase): - def test1(self) -> None: """ Test rendering a basic image code block. @@ -682,7 +679,6 @@ def test_mermaid7(self) -> None: reason="Disabled because of CmampTask10710", ) class Test_render_images2(hunitest.TestCase): - def helper(self, file_name: str) -> None: """ Helper function to test rendering images from a file. diff --git a/dev_scripts_helpers/documentation/test/test_transform_notes.py b/dev_scripts_helpers/documentation/test/test_transform_notes.py index 8931d4441..b24c01c9d 100644 --- a/dev_scripts_helpers/documentation/test/test_transform_notes.py +++ b/dev_scripts_helpers/documentation/test/test_transform_notes.py @@ -20,7 +20,6 @@ reason="Disabled because of CmampTask10710", ) class Test_markdown_to_latex1(hunitest.TestCase): - def test1(self) -> None: """ Test a simple nested list with no frame title. diff --git a/dev_scripts_helpers/documentation/transform_notes.py b/dev_scripts_helpers/documentation/transform_notes.py index a4b41eb0f..43a6916ab 100755 --- a/dev_scripts_helpers/documentation/transform_notes.py +++ b/dev_scripts_helpers/documentation/transform_notes.py @@ -41,7 +41,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_input_output_args( parser, diff --git a/dev_scripts_helpers/encrypt_models/encrypt_model.py b/dev_scripts_helpers/encrypt_models/encrypt_model.py index 52735c1ed..3355b2581 100755 --- a/dev_scripts_helpers/encrypt_models/encrypt_model.py +++ b/dev_scripts_helpers/encrypt_models/encrypt_model.py @@ -21,6 +21,7 @@ --model_dag_builder_file "C5/C5a_pipeline.py \ -v DEBUG """ + import argparse import logging import os @@ -102,7 +103,7 @@ def _encrypt_input_dir( cmd = f"docker build -f {temp_dockerfile.name} -t {docker_image_tag} --build-arg user_id={user_id} --build-arg group_id={group_id} ." (_, output) = hsystem.system_to_string(cmd) _LOG.debug(output) - #os.remove(temp_dockerfile_path) + # os.remove(temp_dockerfile_path) # Run Docker container to encrypt the model. _LOG.info("Running Docker container.") work_dir = os.getcwd() @@ -110,7 +111,9 @@ def _encrypt_input_dir( mount = f"type=bind,source={work_dir},target={docker_output_dir}" # Use `-i` option to store the runtime folder inside the target dir, this way we do not # need to tweak init files. - encryption_flow = f"pyarmor gen -i --recursive {input_dir} --output {output_dir}" + encryption_flow = ( + f"pyarmor gen -i --recursive {input_dir} --output {output_dir}" + ) if docker_build_target is not None: docker_cmd = f"docker run --rm -it --user {user_id}:{group_id} --platform {docker_build_target} --workdir {docker_output_dir} --mount {mount} {docker_image_tag} {encryption_flow}" else: @@ -119,12 +122,10 @@ def _encrypt_input_dir( _LOG.debug(output) # Check that command worked by ensuring that there are dirs in the target dir. n_files = len(os.listdir(output_dir)) - hdbg.dassert_lt( - 0, n_files, "No files in output_dir=`%s`", output_dir - ) - # For some reason `pyarmor` encrypts the data to `encrypted_pipelines/pipelines` + hdbg.dassert_lt(0, n_files, "No files in output_dir=`%s`", output_dir) + # For some reason `pyarmor` encrypts the data to `encrypted_pipelines/pipelines` # instead of just `encrypted_pipelines`, same with `core_lem`, e.g., `encrypted_core_lem/core_lem` - # instead of just `core_lem`. + # instead of just `core_lem`. # TODO(Grisha): can we fix that on pyarmor level? # E.g., extract `pipelines` from `dataflow_lemonade/pipelines` and `core_lem` from `core_lem`. last_dir = os.path.basename(os.path.normpath(input_dir)) @@ -136,6 +137,7 @@ def _encrypt_input_dir( ) return output_dir + # TODO(gp): Add a function to check that everything is encrypted. # find dataflow_lemonade/encrypted_pipelines -name "*.py" | grep -v pytransform | xargs -n 1 cat | grep -v "\x" @@ -209,7 +211,7 @@ def _test_model(model_dag_builder: str, model_dag_builder_file: str) -> None: cmd = f"invoke docker_cmd -c 'bash {temp_file_path}'" _, output = hsystem.system_to_string(cmd) _LOG.debug(output) - #os.remove(temp_file_path) + # os.remove(temp_file_path) def _test_models_in_dir(input_dir: str, model_dag_builder: str) -> None: @@ -227,7 +229,8 @@ def _test_models_in_dir(input_dir: str, model_dag_builder: str) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--input_dir", required=True, type=str, help="Source model directory" @@ -282,8 +285,10 @@ def _parse() -> argparse.ArgumentParser: # TODO(gp): Split this code in function, one per function. def _main(parser: argparse.ArgumentParser) -> None: - hdbg.dassert(not hserver.is_inside_docker(), - "This script runs outside the dev container and in the thin environment") + hdbg.dassert( + not hserver.is_inside_docker(), + "This script runs outside the dev container and in the thin environment", + ) args = parser.parse_args() hdbg.init_logger(verbosity=args.log_level, use_exec_path=True) # @@ -293,8 +298,10 @@ def _main(parser: argparse.ArgumentParser) -> None: output_dir = args.output_dir # If `output_dir` is not specified, use the parent directory of `input_dir`. if output_dir is None: - output_dir = os.path.join(os.path.dirname(input_dir), - "encrypted_" + os.path.basename(input_dir)) + output_dir = os.path.join( + os.path.dirname(input_dir), + "encrypted_" + os.path.basename(input_dir), + ) _LOG.info("output_dir=%s", output_dir) # Get the DAG builder. if args.test: @@ -317,8 +324,8 @@ def _main(parser: argparse.ArgumentParser) -> None: _encrypt_input_dir( input_dir, output_dir, args.build_target, args.docker_image_tag ) - # TODO(Grisha): in `pyarmor v9.x` tweaking init files does not seem necessary, - # make sure to utilize `-i` option when running `pyarmor gen ...` this way it + # TODO(Grisha): in `pyarmor v9.x` tweaking init files does not seem necessary, + # make sure to utilize `-i` option when running `pyarmor gen ...` this way it # will put the runtime folder inside the target dir and this step is not needed; # keeping it here just as a reference. # 3) Tweak `__init__.py` file. @@ -349,7 +356,9 @@ def _main(parser: argparse.ArgumentParser) -> None: # Add trailing slashes so that the machine treats them as folders. # Use the `--delete` option to delete all files from target that are not in source; we # assume that the source repo contains the latest state. - _, output = hsystem.system_to_string(f"rsync -a --delete {input_dir_tmp}/ {output_dir_tmp}/") + _, output = hsystem.system_to_string( + f"rsync -a --delete {input_dir_tmp}/ {output_dir_tmp}/" + ) # ls /data/saggese/src_vc/lemonade1/dataflow_lemonade/encrypted_pipelines cmd = f"ls {input_dir_tmp}" hsystem.system(cmd, suppress_output=False) diff --git a/dev_scripts_helpers/git/gd_notebook.py b/dev_scripts_helpers/git/gd_notebook.py index a5cf23369..e1e1103e4 100755 --- a/dev_scripts_helpers/git/gd_notebook.py +++ b/dev_scripts_helpers/git/gd_notebook.py @@ -135,7 +135,8 @@ def _get_files(args: argparse.Namespace) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("-f", "--files", nargs="+", help="Files to process") parser.add_argument( @@ -208,7 +209,9 @@ def _main(parser: argparse.ArgumentParser) -> None: _LOG.info( "file_names=%s\n%s", len(file_names), - "\n".join(["%s -> %s" % (cwd_f, abs_f) for (abs_f, cwd_f) in file_names]), + "\n".join( + ["%s -> %s" % (cwd_f, abs_f) for (abs_f, cwd_f) in file_names] + ), ) hdbg.dassert_lte(1, len(file_names)) # Create tmp dir. diff --git a/dev_scripts_helpers/git/git_hooks/install_hooks.py b/dev_scripts_helpers/git/git_hooks/install_hooks.py index 106d553a6..eebd5d163 100755 --- a/dev_scripts_helpers/git/git_hooks/install_hooks.py +++ b/dev_scripts_helpers/git/git_hooks/install_hooks.py @@ -37,7 +37,8 @@ def _main() -> None: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--action", diff --git a/dev_scripts_helpers/git/git_hooks/pre-commit-dry-run.py b/dev_scripts_helpers/git/git_hooks/pre-commit-dry-run.py index 9c9e80a4f..d8babb33e 100755 --- a/dev_scripts_helpers/git/git_hooks/pre-commit-dry-run.py +++ b/dev_scripts_helpers/git/git_hooks/pre-commit-dry-run.py @@ -25,7 +25,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) # parser.add_argument("--text", action="store", type=str, required=True) # parser.add_argument("--step", action="store", type=int, required=True) @@ -39,7 +40,9 @@ def _main(parser: argparse.ArgumentParser) -> None: # _LOG.warning("\n%s", hprint.frame("This is a dry run!")) git_root = hgit.get_client_root(super_module=False) - cmd = rf'''cd {git_root} && find . -type f -name "*" -not -path "*/\.git/*"''' + cmd = ( + rf'''cd {git_root} && find . -type f -name "*" -not -path "*/\.git/*"''' + ) _, file_list = hsystem.system_to_string(cmd) file_list = file_list.split("\n") # diff --git a/dev_scripts_helpers/git/git_hooks/translate.py b/dev_scripts_helpers/git/git_hooks/translate.py index c03de0c98..5fc6540a6 100755 --- a/dev_scripts_helpers/git/git_hooks/translate.py +++ b/dev_scripts_helpers/git/git_hooks/translate.py @@ -20,7 +20,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--text", action="store", type=str, required=True) parser.add_argument("--step", action="store", type=int, required=True) diff --git a/dev_scripts_helpers/git/git_submodules.py b/dev_scripts_helpers/git/git_submodules.py index e46c0312c..f3c5fec2c 100755 --- a/dev_scripts_helpers/git/git_submodules.py +++ b/dev_scripts_helpers/git/git_submodules.py @@ -71,7 +71,8 @@ def _roll_fwd(dir_names: List[str], auto_commit: bool, short_hash: bool) -> None def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--pull", action="store_true") parser.add_argument("--show", action="store_true") diff --git a/dev_scripts_helpers/git/gsp.py b/dev_scripts_helpers/git/gsp.py index 1560f6911..50bd132fb 100755 --- a/dev_scripts_helpers/git/gsp.py +++ b/dev_scripts_helpers/git/gsp.py @@ -55,7 +55,8 @@ def _main(parser: argparse.ArgumentParser) -> None: def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "-m", default=None, dest="message", help="Add message to commit" diff --git a/dev_scripts_helpers/git/gup.py b/dev_scripts_helpers/git/gup.py index 6dcddf632..7b2df339b 100755 --- a/dev_scripts_helpers/git/gup.py +++ b/dev_scripts_helpers/git/gup.py @@ -67,7 +67,8 @@ def _main(parser: argparse.ArgumentParser) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_verbosity_arg(parser) return parser diff --git a/dev_scripts_helpers/github/dockerized_sync_gh_issue_labels.py b/dev_scripts_helpers/github/dockerized_sync_gh_issue_labels.py index 033f764f9..0944a400d 100755 --- a/dev_scripts_helpers/github/dockerized_sync_gh_issue_labels.py +++ b/dev_scripts_helpers/github/dockerized_sync_gh_issue_labels.py @@ -43,7 +43,6 @@ class Label: - def __init__(self, name: str, description: str, color: str): """ Initialize the label with name, description, and color. @@ -134,7 +133,8 @@ def to_dict(self) -> Dict[str, str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_verbosity_arg(parser) parser.add_argument( diff --git a/dev_scripts_helpers/github/dockerized_sync_gh_repo_settings.py b/dev_scripts_helpers/github/dockerized_sync_gh_repo_settings.py index 8ff0ba5e4..4e035f0ac 100644 --- a/dev_scripts_helpers/github/dockerized_sync_gh_repo_settings.py +++ b/dev_scripts_helpers/github/dockerized_sync_gh_repo_settings.py @@ -76,7 +76,6 @@ class _RepoAndBranchSettings: - def __init__(self, repo_and_branch_settings: Dict[str, Any]): """ Initialize a nested dictionary of branch protection and repository @@ -231,7 +230,8 @@ def get_branch_protection_settings( dismissal_restrictions = {} if dismissal: dismissal_restrictions["users"] = [ - user.login for user in getattr(dismissal, "users", []) + user.login + for user in getattr(dismissal, "users", []) ] dismissal_restrictions["teams"] = [ team.name for team in getattr(dismissal, "teams", []) @@ -252,7 +252,9 @@ def get_branch_protection_settings( # Package all the information in a dictionary for the current # branch. branch_protection[branch.name] = { - "enforce_admins": getattr(protection, "enforce_admins", None), + "enforce_admins": getattr( + protection, "enforce_admins", None + ), "allow_force_pushes": getattr( protection, "allow_force_pushes", None ), @@ -723,7 +725,8 @@ def _parse() -> argparse.ArgumentParser: Parse command line arguments. """ parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) # Create subparsers for different commands. subparsers = parser.add_subparsers(dest="command", help="Sub-command help") diff --git a/dev_scripts_helpers/github/set_secrets_and_variables.py b/dev_scripts_helpers/github/set_secrets_and_variables.py index b1ab0e4be..50c784aad 100755 --- a/dev_scripts_helpers/github/set_secrets_and_variables.py +++ b/dev_scripts_helpers/github/set_secrets_and_variables.py @@ -40,7 +40,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_verbosity_arg(parser) parser.add_argument( diff --git a/dev_scripts_helpers/github/sync_gh_issue_labels.py b/dev_scripts_helpers/github/sync_gh_issue_labels.py index 7dd20ffa0..74c0f7b64 100755 --- a/dev_scripts_helpers/github/sync_gh_issue_labels.py +++ b/dev_scripts_helpers/github/sync_gh_issue_labels.py @@ -33,7 +33,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--input_file", diff --git a/dev_scripts_helpers/github/sync_gh_projects.py b/dev_scripts_helpers/github/sync_gh_projects.py index 34f268fe8..4ccd977f4 100755 --- a/dev_scripts_helpers/github/sync_gh_projects.py +++ b/dev_scripts_helpers/github/sync_gh_projects.py @@ -265,7 +265,8 @@ def _parse() -> argparse.ArgumentParser: :return: configured ArgumentParser instance. """ p = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) p.add_argument( "--owner", diff --git a/dev_scripts_helpers/github/test/test_dockerized_sync_gh_issue_labels.py b/dev_scripts_helpers/github/test/test_dockerized_sync_gh_issue_labels.py index 1356bf935..ad193916b 100644 --- a/dev_scripts_helpers/github/test/test_dockerized_sync_gh_issue_labels.py +++ b/dev_scripts_helpers/github/test/test_dockerized_sync_gh_issue_labels.py @@ -44,7 +44,6 @@ def _make_mock_label(label: dshgdsgil.Label) -> umock.Mock: class TestLabel1(hunitest.TestCase): - def test_save_label(self) -> None: """ Test saving a label object to a file. @@ -102,7 +101,6 @@ def test_load_label(self) -> None: class TestDockerizedSyncGitHubIssueLabels(hunitest.TestCase): - @pytest.fixture(autouse=True) def setup_teardown_test(self) -> Generator: # Run before each test. diff --git a/dev_scripts_helpers/github/test/test_sync_gh_issue_labels.py b/dev_scripts_helpers/github/test/test_sync_gh_issue_labels.py index b6dd90f16..cde97e743 100644 --- a/dev_scripts_helpers/github/test/test_sync_gh_issue_labels.py +++ b/dev_scripts_helpers/github/test/test_sync_gh_issue_labels.py @@ -19,7 +19,6 @@ class Test_sync_gh_issue_labels1(hunitest.TestCase): - @pytest.mark.skip("Enable after HelpersTask753") # TODO(sandeep): Check the dockerized executable once the CmampTask10710 is fixed. @pytest.mark.skipif( diff --git a/dev_scripts_helpers/infra/gdrive.py b/dev_scripts_helpers/infra/gdrive.py index 6b324d702..e5976fcc2 100755 --- a/dev_scripts_helpers/infra/gdrive.py +++ b/dev_scripts_helpers/infra/gdrive.py @@ -126,7 +126,8 @@ def _rclone_copy_to_gdrive(local_src_dir, remote_dst_dir, log_dir, dry_run): def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--action", diff --git a/dev_scripts_helpers/infra/old/ssh_tunnels.py b/dev_scripts_helpers/infra/old/ssh_tunnels.py index 1ded0e189..2bd6530f8 100755 --- a/dev_scripts_helpers/infra/old/ssh_tunnels.py +++ b/dev_scripts_helpers/infra/old/ssh_tunnels.py @@ -39,7 +39,8 @@ def _main() -> None: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) _help = """ - start: start a notebook if no notebook server is running at the requested port diff --git a/dev_scripts_helpers/lib_tasks_data_qa.py b/dev_scripts_helpers/lib_tasks_data_qa.py index c0b70757d..0f23eafcf 100644 --- a/dev_scripts_helpers/lib_tasks_data_qa.py +++ b/dev_scripts_helpers/lib_tasks_data_qa.py @@ -260,7 +260,9 @@ def run_single_dataset_qa_notebook( if dataset_signature_as_dict["data_type"] == "ohlcv": notebook_path = "amp/datapull/ccxt/data/qa/notebooks/data_qa_ohlcv.ipynb" elif dataset_signature_as_dict["data_type"] == "bid_ask": - notebook_path = "amp/datapull/ccxt/data/qa/notebooks/data_qa_bid_ask.ipynb" + notebook_path = ( + "amp/datapull/ccxt/data/qa/notebooks/data_qa_bid_ask.ipynb" + ) else: raise NotImplementedError _run_data_qa_notebook( @@ -307,9 +309,7 @@ def run_cross_dataset_qa_notebook( _ = ctx # TODO(Juraj): come up with a more modular solution to executing the correct notebook. data_type = "bid_ask" if "bid_ask" in dataset_signature1 else "ohlcv" - notebook_path = ( - f"amp/datapull/common/data/qa/notebooks/cross_dataset_qa_{data_type}.ipynb" - ) + notebook_path = f"amp/datapull/common/data/qa/notebooks/cross_dataset_qa_{data_type}.ipynb" _run_data_qa_notebook( config_dict, base_dst_dir, notebook_path, s3_publish_dst_dir ) diff --git a/dev_scripts_helpers/llms/ai_review.py b/dev_scripts_helpers/llms/ai_review.py index 66f8671c4..3a4cd1653 100755 --- a/dev_scripts_helpers/llms/ai_review.py +++ b/dev_scripts_helpers/llms/ai_review.py @@ -27,7 +27,6 @@ # - run the script to process input and write transformed output # - run the script to process input and extract a cfile - import argparse import logging import os diff --git a/dev_scripts_helpers/llms/dockerized_llm_review.py b/dev_scripts_helpers/llms/dockerized_llm_review.py index f6db1739d..70288d6b1 100755 --- a/dev_scripts_helpers/llms/dockerized_llm_review.py +++ b/dev_scripts_helpers/llms/dockerized_llm_review.py @@ -23,7 +23,9 @@ _LOG = logging.getLogger(__name__) -def _load_review_guidelines(guidelines_doc_filename: str) -> Dict[str, List[str]]: +def _load_review_guidelines( + guidelines_doc_filename: str, +) -> Dict[str, List[str]]: """ Load automated review guidelines. diff --git a/dev_scripts_helpers/llms/llm_apply_cfile.py b/dev_scripts_helpers/llms/llm_apply_cfile.py index 55dce622f..6e60e7457 100755 --- a/dev_scripts_helpers/llms/llm_apply_cfile.py +++ b/dev_scripts_helpers/llms/llm_apply_cfile.py @@ -140,7 +140,9 @@ def _run_dockerized_llm_apply_cfile( ] ) docker_cmd = " ".join(docker_cmd) - ret = hdocker.process_docker_cmd(docker_cmd, container_image, dockerfile, mode) + ret = hdocker.process_docker_cmd( + docker_cmd, container_image, dockerfile, mode + ) return ret diff --git a/dev_scripts_helpers/llms/llm_prompts.py b/dev_scripts_helpers/llms/llm_prompts.py index 59ff4ecbf..7408e7e45 100644 --- a/dev_scripts_helpers/llms/llm_prompts.py +++ b/dev_scripts_helpers/llms/llm_prompts.py @@ -1312,7 +1312,9 @@ def _convert_to_vim_cfile_str(txt: str, in_file_name: str) -> str: return txt_out -def _convert_to_vim_cfile(txt: str, in_file_name: str, out_file_name: str) -> str: +def _convert_to_vim_cfile( + txt: str, in_file_name: str, out_file_name: str +) -> str: """ Convert the text passed to a vim cfile. @@ -1374,9 +1376,12 @@ def run_prompt( prompt_tags = get_prompt_tags() hdbg.dassert_in(prompt_tag, prompt_tags) python_cmd = f"{prompt_tag}()" - system_prompt, pre_transforms, post_transforms, post_container_transforms = ( - eval(python_cmd) - ) + ( + system_prompt, + pre_transforms, + post_transforms, + post_container_transforms, + ) = eval(python_cmd) # Check return types. hdbg.dassert_isinstance(system_prompt, str) hdbg.dassert_isinstance(pre_transforms, set) diff --git a/dev_scripts_helpers/llms/llm_review.py b/dev_scripts_helpers/llms/llm_review.py index 34835a8c3..d4868524c 100755 --- a/dev_scripts_helpers/llms/llm_review.py +++ b/dev_scripts_helpers/llms/llm_review.py @@ -184,7 +184,9 @@ def _parse() -> argparse.ArgumentParser: action="store_true", help="Select files modified in the current branch with respect to master", ) - parser.add_argument("--skip_files", nargs="+", type=str, help="Files to skip") + parser.add_argument( + "--skip_files", nargs="+", type=str, help="Files to skip" + ) # Reviewer guidelines file. parser.add_argument( "--guidelines_doc_filename", @@ -218,7 +220,9 @@ def _main(parser: argparse.ArgumentParser) -> None: args.branch, ) _LOG.debug( - "Reviewing %s files; file_paths=%s", len(file_paths), " ".join(file_paths) + "Reviewing %s files; file_paths=%s", + len(file_paths), + " ".join(file_paths), ) _run_dockerized_llm_review( file_paths, diff --git a/dev_scripts_helpers/llms/llm_transform.py b/dev_scripts_helpers/llms/llm_transform.py index 74c89e68c..4901c5184 100755 --- a/dev_scripts_helpers/llms/llm_transform.py +++ b/dev_scripts_helpers/llms/llm_transform.py @@ -27,7 +27,6 @@ # - run the script to process input and write transformed output # - run the script to process input and extract a cfile - import argparse import logging import os diff --git a/dev_scripts_helpers/llms/test/test_llm_prompts.py b/dev_scripts_helpers/llms/test/test_llm_prompts.py index 9537fa2a7..9ae879d4a 100644 --- a/dev_scripts_helpers/llms/test/test_llm_prompts.py +++ b/dev_scripts_helpers/llms/test/test_llm_prompts.py @@ -20,7 +20,6 @@ reason="Disabled because of CmampTask10710", ) class Test_convert_to_vim_cfile1(hunitest.TestCase): - def test1(self) -> None: """ Test converting a simple error message to vim cfile format. @@ -55,7 +54,6 @@ def test2(self) -> None: reason="Disabled because of CmampTask10710", ) class Test_prompt_tags1(hunitest.TestCase): - def test1(self) -> None: prompt_tags = dshlllpr.get_prompt_tags() _LOG.debug(hprint.to_str("prompt_tags")) @@ -74,7 +72,6 @@ def test1(self) -> None: reason="Disabled because of CmampTask10710", ) class Test_run_prompt1(hunitest.TestCase): - # TODO(gp): Add one tests for each prompt. def test_code_fix_from_imports1(self) -> None: diff --git a/dev_scripts_helpers/llms/test/test_llm_transform.py b/dev_scripts_helpers/llms/test/test_llm_transform.py index 6f4658ffb..da63a52bc 100644 --- a/dev_scripts_helpers/llms/test/test_llm_transform.py +++ b/dev_scripts_helpers/llms/test/test_llm_transform.py @@ -137,8 +137,7 @@ def test_all_prompts1(self) -> None: hdbg.dassert(not os.path.exists(out_file_name)) # Run the test. cmd = ( - f"{script} -i {in_file_name} -o {out_file_name}" - f" -p {prompt_tag}" + f"{script} -i {in_file_name} -o {out_file_name} -p {prompt_tag}" ) hsystem.system(cmd) # Check. diff --git a/dev_scripts_helpers/misc/extract_bounties.py b/dev_scripts_helpers/misc/extract_bounties.py index 9a6212f18..2d94f8b99 100755 --- a/dev_scripts_helpers/misc/extract_bounties.py +++ b/dev_scripts_helpers/misc/extract_bounties.py @@ -42,7 +42,8 @@ def _parse() -> argparse.ArgumentParser: :return: argument parser with all command-line arguments """ parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--input_file", diff --git a/dev_scripts_helpers/misc/get_url_titles.py b/dev_scripts_helpers/misc/get_url_titles.py index f58b8cec9..832bdd529 100644 --- a/dev_scripts_helpers/misc/get_url_titles.py +++ b/dev_scripts_helpers/misc/get_url_titles.py @@ -23,7 +23,6 @@ def get_page_title(url: str) -> str: class TitleParser(HTMLParser): - def __init__(self) -> None: super().__init__() self.in_title: bool = False diff --git a/dev_scripts_helpers/notebooks/add_toc_to_notebook.py b/dev_scripts_helpers/notebooks/add_toc_to_notebook.py index 620dd1514..eb8900f8b 100755 --- a/dev_scripts_helpers/notebooks/add_toc_to_notebook.py +++ b/dev_scripts_helpers/notebooks/add_toc_to_notebook.py @@ -108,7 +108,8 @@ def add_toc(file_name: str) -> dict: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) group = parser.add_mutually_exclusive_group(required=True) group.add_argument( diff --git a/dev_scripts_helpers/notebooks/dockerized_extract_notebook_images.py b/dev_scripts_helpers/notebooks/dockerized_extract_notebook_images.py index fd77c7eb7..4a50bc636 100755 --- a/dev_scripts_helpers/notebooks/dockerized_extract_notebook_images.py +++ b/dev_scripts_helpers/notebooks/dockerized_extract_notebook_images.py @@ -42,7 +42,9 @@ def __init__(self, notebook_path: str, output_dir: str) -> None: self.output_dir = output_dir @staticmethod - def _extract_regions_from_notebook(notebook_path: str) -> List[Tuple[str, str, List]]: + def _extract_regions_from_notebook( + notebook_path: str, + ) -> List[Tuple[str, str, List]]: """ Extract regions from a notebook based on extraction markers. @@ -117,7 +119,8 @@ def test_func(): hdbg.dassert( not in_extract, "Found a start marker while in an extraction region at cell %s\n%s", - cell_idx, cell.source + cell_idx, + cell.source, ) # A start marker was found. # Capture the mode and output filename @@ -138,7 +141,8 @@ def test_func(): hdbg.dassert( in_extract, "Found an end marker while not in an extraction region at cell %s\n%s", - cell_idx, cell.source + cell_idx, + cell.source, ) current_cells.append(cell) regions.append( @@ -212,7 +216,7 @@ def extract_and_capture(self) -> list: - Convert the region to an HTML file - Capture a screenshot using Playwright - Clean up the temporary HTML file - + Screenshots are saved in the "screenshots" folder with filenames based on the name provided in the extraction marker. If a name is repeated, a counter suffix (_1, _2, etc.) is appended to ensure unique filenames. A @@ -292,5 +296,6 @@ def _main(parser: argparse.ArgumentParser) -> None: extractor.extract_and_capture() _LOG.info("Extraction completed. Images saved in '%s'", args.out_image_dir) + if __name__ == "__main__": - _main(_parse()) \ No newline at end of file + _main(_parse()) diff --git a/dev_scripts_helpers/notebooks/extract_notebook_images.py b/dev_scripts_helpers/notebooks/extract_notebook_images.py index cf48a2b48..984e498f1 100755 --- a/dev_scripts_helpers/notebooks/extract_notebook_images.py +++ b/dev_scripts_helpers/notebooks/extract_notebook_images.py @@ -30,7 +30,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--in_notebook_filename", diff --git a/dev_scripts_helpers/notebooks/process_jupytext.py b/dev_scripts_helpers/notebooks/process_jupytext.py index 8b068897d..4c6c3958f 100755 --- a/dev_scripts_helpers/notebooks/process_jupytext.py +++ b/dev_scripts_helpers/notebooks/process_jupytext.py @@ -148,7 +148,8 @@ def _test(file_name: str, action: str) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "-f", diff --git a/dev_scripts_helpers/notebooks/publish_notebook.py b/dev_scripts_helpers/notebooks/publish_notebook.py index 4e07c1122..8f0476396 100755 --- a/dev_scripts_helpers/notebooks/publish_notebook.py +++ b/dev_scripts_helpers/notebooks/publish_notebook.py @@ -302,7 +302,9 @@ def _main(parser: argparse.ArgumentParser) -> None: if args.action == "convert": # Convert to HTML. dst_dir = "." - html_file_name = _export_notebook_to_dir(src_file_name, args.tag, dst_dir) + html_file_name = _export_notebook_to_dir( + src_file_name, args.tag, dst_dir + ) # Try to open. hopen.open_file(html_file_name) # Exit the `convert` action. diff --git a/dev_scripts_helpers/notebooks/run_jupyter_server.py b/dev_scripts_helpers/notebooks/run_jupyter_server.py index 0872f68a4..2baab7101 100755 --- a/dev_scripts_helpers/notebooks/run_jupyter_server.py +++ b/dev_scripts_helpers/notebooks/run_jupyter_server.py @@ -61,7 +61,7 @@ def _start(port, action): pids, txt = _get_port_process(port) if pids: _LOG.warning( - "Found other jupyter notebooks running on the same " "port:\n%s", + "Found other jupyter notebooks running on the same port:\n%s", "\n".join(txt), ) if action == "force_start": @@ -87,7 +87,8 @@ def _start(port, action): def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) _help = """ - start: start a notebook if no notebook server is running at the requested port @@ -105,7 +106,7 @@ def _parse() -> argparse.ArgumentParser: "--port", action="store", default=None, - help="Override the " "default port to use", + help="Override the default port to use", ) hparser.add_verbosity_arg(parser) return parser diff --git a/dev_scripts_helpers/notebooks/run_notebook.py b/dev_scripts_helpers/notebooks/run_notebook.py index 01c46ab00..5c8cafb9d 100755 --- a/dev_scripts_helpers/notebooks/run_notebook.py +++ b/dev_scripts_helpers/notebooks/run_notebook.py @@ -147,7 +147,9 @@ def _run_notebook( # TODO(gp): Look for the script. amp_dir = hgit.get_amp_abs_path() script_path = os.path.join( - amp_dir, "helpers_root/dev_scripts_helpers/notebooks", "publish_notebook.py" + amp_dir, + "helpers_root/dev_scripts_helpers/notebooks", + "publish_notebook.py", ) cmd = ( f"python {script_path}" @@ -209,7 +211,8 @@ def _get_workload(args: argparse.Namespace) -> hjoblib.Workload: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) # Add common experiment options. parser = dtfbdtfbaut.add_run_experiment_args(parser, dst_dir_required=True) diff --git a/dev_scripts_helpers/notebooks/test/test_dockerized_extract_notebook_images.py b/dev_scripts_helpers/notebooks/test/test_dockerized_extract_notebook_images.py index 816e7a2aa..8c5674160 100644 --- a/dev_scripts_helpers/notebooks/test/test_dockerized_extract_notebook_images.py +++ b/dev_scripts_helpers/notebooks/test/test_dockerized_extract_notebook_images.py @@ -2,7 +2,6 @@ import os import helpers.hunit_test as hunitest -import helpers.hserver as hserver import dev_scripts_helpers.notebooks.dockerized_extract_notebook_images as dshndb _LOG = logging.getLogger(__name__) @@ -13,9 +12,10 @@ class TestNotebookImageExtractor1(hunitest.TestCase): - def test1(self) -> None: input_dir = self.get_input_dir() src_test_notebook = os.path.join(input_dir, "test_images.ipynb") - act = dshndb._NotebookImageExtractor._extract_regions_from_notebook(src_test_notebook) + act = dshndb._NotebookImageExtractor._extract_regions_from_notebook( + src_test_notebook + ) print(act) diff --git a/dev_scripts_helpers/notebooks/test/test_extract_notebook_images.py b/dev_scripts_helpers/notebooks/test/test_extract_notebook_images.py index 977ff924c..61d4d462e 100644 --- a/dev_scripts_helpers/notebooks/test/test_extract_notebook_images.py +++ b/dev_scripts_helpers/notebooks/test/test_extract_notebook_images.py @@ -5,7 +5,6 @@ import helpers.hunit_test as hunitest import helpers.hserver as hserver import dev_scripts_helpers.notebooks.extract_notebook_images as dshnbe -import dev_scripts_helpers.notebooks.dockerized_extract_notebook_images as dshndb _LOG = logging.getLogger(__name__) @@ -20,7 +19,6 @@ ) @pytest.mark.superslow("~42 sec.") class Test_run_dockerized_notebook_image_extractor1(hunitest.TestCase): - def test1(self) -> None: """ Test the `run_dockerized_notebook_image_extractor` function. diff --git a/dev_scripts_helpers/old/create_conda/_setenv_amp.py b/dev_scripts_helpers/old/create_conda/_setenv_amp.py index a7e424f40..5ed0e5348 100755 --- a/dev_scripts_helpers/old/create_conda/_setenv_amp.py +++ b/dev_scripts_helpers/old/create_conda/_setenv_amp.py @@ -90,7 +90,8 @@ def _main(parser: argparse.ArgumentParser) -> None: # - Config PATH. # dirs = [ - f"{amp_path}/dev_scripts/{d}" for d in dsoccseli.get_dev_scripts_subdirs() + f"{amp_path}/dev_scripts/{d}" + for d in dsoccseli.get_dev_scripts_subdirs() ] dirs.append(os.path.join(amp_path, "documentation", "scripts")) dsoccseli.config_path(dirs, txt) diff --git a/dev_scripts_helpers/old/create_conda/_setenv_lib.py b/dev_scripts_helpers/old/create_conda/_setenv_lib.py index 2248bf293..407b594bf 100644 --- a/dev_scripts_helpers/old/create_conda/_setenv_lib.py +++ b/dev_scripts_helpers/old/create_conda/_setenv_lib.py @@ -266,10 +266,14 @@ def save_script(args: argparse.Namespace, txt: List[str]) -> None: def parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( - "-o", "--output_file", default=None, help="File to write. None for stdout" + "-o", + "--output_file", + default=None, + help="File to write. None for stdout", ) parser.add_argument( "-e", "--conda_env", default=None, help="Select the conda env to use." diff --git a/dev_scripts_helpers/old/create_conda/install/create_conda.py b/dev_scripts_helpers/old/create_conda/install/create_conda.py index f9f96459c..a5b5b3802 100755 --- a/dev_scripts_helpers/old/create_conda/install/create_conda.py +++ b/dev_scripts_helpers/old/create_conda/install/create_conda.py @@ -270,7 +270,9 @@ def _run_pip_install(args: Any, conda_env_name: str) -> None: def _test_conda_env(conda_env_name: str) -> None: # Test activating. - _LOG.info("\n%s", hprint.frame(f"Test activate conda env '{conda_env_name}'")) + _LOG.info( + "\n%s", hprint.frame(f"Test activate conda env '{conda_env_name}'") + ) cmd = f"conda activate {conda_env_name} && conda info --envs" holdcond.conda_system(cmd, suppress_output=False) # Check packages. @@ -287,7 +289,8 @@ def _test_conda_env(conda_env_name: str) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--delete_env_if_exists", action="store_true") parser.add_argument( diff --git a/dev_scripts_helpers/old/create_conda/install/print_conda_packages.py b/dev_scripts_helpers/old/create_conda/install/print_conda_packages.py index 8bcff6c25..d5b7c09dc 100755 --- a/dev_scripts_helpers/old/create_conda/install/print_conda_packages.py +++ b/dev_scripts_helpers/old/create_conda/install/print_conda_packages.py @@ -18,7 +18,8 @@ def _main() -> None: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_verbosity_arg(parser) parser.add_argument( diff --git a/dev_scripts_helpers/old/linter/linter.py b/dev_scripts_helpers/old/linter/linter.py index d1c6d97fe..c6f828f27 100755 --- a/dev_scripts_helpers/old/linter/linter.py +++ b/dev_scripts_helpers/old/linter/linter.py @@ -1208,7 +1208,7 @@ def _check_import(file_name: str, line_num: int, line: str) -> str: m = re.match(r"\s*from\s+(\S+)\s+import\s+.*", line) if m: if m.group(1) != "typing": - msg = "%s:%s: do not use '%s' use 'import foo.bar " "as fba'" % ( + msg = "%s:%s: do not use '%s' use 'import foo.bar as fba'" % ( file_name, line_num, line.rstrip().lstrip(), @@ -1300,7 +1300,9 @@ def _check_line_by_line( # Look for conflicts markers. if _CustomPythonChecks.DEBUG: _LOG.debug("* Look for conflict markers") - if any(line.startswith(c) for c in ["<<<<<<<", "=======", ">>>>>>>"]): + if any( + line.startswith(c) for c in ["<<<<<<<", "=======", ">>>>>>>"] + ): msg = "%s:%s: there are conflict markers" % (file_name, i + 1) output.append(msg) # Format separating lines. @@ -1371,7 +1373,9 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: ext = os.path.splitext(file_name)[1] output: List[str] = [] if ext not in (".txt", ".md"): - _LOG.debug("Skipping file_name='%s' because ext='%s'", file_name, ext) + _LOG.debug( + "Skipping file_name='%s' because ext='%s'", file_name, ext + ) return output # Run lint_notes.py. executable = "lint_notes.py" @@ -1745,7 +1749,8 @@ def _main(args: argparse.Namespace) -> int: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) # Select files. parser.add_argument( @@ -1810,7 +1815,9 @@ def _parse() -> argparse.ArgumentParser: help="Process only python scripts excluding paired notebooks", ) parser.add_argument( - "--only_ipynb", action="store_true", help="Process only jupyter notebooks" + "--only_ipynb", + action="store_true", + help="Process only jupyter notebooks", ) parser.add_argument( "--only_paired_jupytext", diff --git a/dev_scripts_helpers/old/linter/linter_master_report.py b/dev_scripts_helpers/old/linter/linter_master_report.py index ebb527a98..01d017da0 100755 --- a/dev_scripts_helpers/old/linter/linter_master_report.py +++ b/dev_scripts_helpers/old/linter/linter_master_report.py @@ -21,7 +21,9 @@ _LOG = logging.getLogger(__name__) -def _perform_linter_for_test_branch(base_commit_sha: str) -> Tuple[int, int, str]: +def _perform_linter_for_test_branch( + base_commit_sha: str, +) -> Tuple[int, int, str]: cmd = "git reset --hard" # Clean up the client from all linter artifacts. hsystem.system(cmd) @@ -171,10 +173,14 @@ def _calculate_stats( # Statuses and additional info message.append(f"- Master (sha: {base_commit_sha})") message.append(f"\t- Number of lints: {master_lints}") - message.append(f"\t- Dirty (i.e., linter was not run): {master_dirty_status}") + message.append( + f"\t- Dirty (i.e., linter was not run): {master_dirty_status}" + ) message.append(f"- Branch ({head_branch_name}: {head_commit_sha})") message.append(f"\t- Number of lints: {branch_lints}") - message.append(f"\t- Dirty (i.e., linter was not run): {branch_dirty_status}") + message.append( + f"\t- Dirty (i.e., linter was not run): {branch_dirty_status}" + ) diff_lints = branch_lints - master_lints message.append( f"\nThe number of lints introduced with this change: {diff_lints}" @@ -188,7 +194,8 @@ def _calculate_stats( def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) # Select files. parser.add_argument( diff --git a/dev_scripts_helpers/old/linter/pre_pr_checklist.py b/dev_scripts_helpers/old/linter/pre_pr_checklist.py index 727bf6d8f..85c043048 100755 --- a/dev_scripts_helpers/old/linter/pre_pr_checklist.py +++ b/dev_scripts_helpers/old/linter/pre_pr_checklist.py @@ -36,7 +36,8 @@ def _print_help(parser: argparse.ArgumentParser) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "-a", diff --git a/dev_scripts_helpers/old/linter/process_jupytext.py b/dev_scripts_helpers/old/linter/process_jupytext.py index 4367444e8..c774392fb 100755 --- a/dev_scripts_helpers/old/linter/process_jupytext.py +++ b/dev_scripts_helpers/old/linter/process_jupytext.py @@ -140,7 +140,8 @@ def _test(file_name: str, action: str) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "-f", diff --git a/dev_scripts_helpers/poetry/run_poetry_debug.py b/dev_scripts_helpers/poetry/run_poetry_debug.py index b90278ecc..9e291d01e 100755 --- a/dev_scripts_helpers/poetry/run_poetry_debug.py +++ b/dev_scripts_helpers/poetry/run_poetry_debug.py @@ -386,7 +386,8 @@ def _get_execution_time_from_log(log_file: str) -> str: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--debug_mode", diff --git a/dev_scripts_helpers/release_sorrentum/filter_repo/git-filter-repo b/dev_scripts_helpers/release_sorrentum/filter_repo/git-filter-repo index ae71fad1d..429ca8d14 100755 --- a/dev_scripts_helpers/release_sorrentum/filter_repo/git-filter-repo +++ b/dev_scripts_helpers/release_sorrentum/filter_repo/git-filter-repo @@ -77,8 +77,8 @@ def setup_gettext(): #locale.bindtextdomain(TEXTDOMAIN, podir); ## but the python docs suggest using the gettext module (which doesn't ## have setlocale()) instead, so: - gettext.textdomain(TEXTDOMAIN); - gettext.bindtextdomain(TEXTDOMAIN, podir); + gettext.textdomain(TEXTDOMAIN) + gettext.bindtextdomain(TEXTDOMAIN, podir) def _timedelta_to_seconds(delta): """ @@ -1107,7 +1107,7 @@ class FastExportParser(object): original_id = None if self._currentline.startswith(b'original-oid'): - original_id = self._parse_original_id(); + original_id = self._parse_original_id() data = self._parse_data() if self._currentline == b'\n': @@ -1186,7 +1186,7 @@ class FastExportParser(object): original_id = None if self._currentline.startswith(b'original-oid'): - original_id = self._parse_original_id(); + original_id = self._parse_original_id() author_name = None author_email = None @@ -1261,7 +1261,7 @@ class FastExportParser(object): # Now print the resulting commit, or if prunable skip it self._latest_orig_commit[branch] = commit.id - if not (commit.old_id or commit.id) in _SKIPPED_COMMITS: + if (commit.old_id or commit.id) not in _SKIPPED_COMMITS: self._latest_commit[branch] = commit.id if not commit.dumped: self._imported_refs.add(commit.branch) @@ -1283,7 +1283,7 @@ class FastExportParser(object): original_id = None if self._currentline.startswith(b'original-oid'): - original_id = self._parse_original_id(); + original_id = self._parse_original_id() tagger_name, tagger_email, tagger_date = None, None, None if self._currentline.startswith(b'tagger'): @@ -2388,7 +2388,7 @@ class RepoAnalyze(object): # to avoid dying on commits with parents that we haven't seen before if args.refs: graph.record_external_commits([p for p in parents - if not p in graph.value]) + if p not in graph.value]) # Analyze this commit and update progress RepoAnalyze.analyze_commit(stats, graph, commit, parents, date, @@ -2905,7 +2905,7 @@ class RepoFilter(object): try: cmd = 'git config remote.origin.url' output = subproc.check_output(cmd.split()).strip() - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: output = None msg = "" if output and os.path.isdir(output): @@ -3285,7 +3285,7 @@ class RepoFilter(object): if ( self._args.replace_text # not (if blob contains zero byte in the first 8Kb, that is, if blob is binary data) - and not b"\0" in blob.data[0:8192] + and b"\0" not in blob.data[0:8192] ): for literal, replacement in self._args.replace_text['literals']: blob.data = blob.data.replace(literal, replacement) @@ -3570,7 +3570,7 @@ class RepoFilter(object): '%s:%s' % (full_branch, decode(marks_basename))] try: contents = subproc.check_output(cmd) - except subprocess.CalledProcessError as e: # pragma: no cover + except subprocess.CalledProcessError: # pragma: no cover raise SystemExit(_("Failed loading %s from %s") % (decode(marks_basename), full_branch)) if contents: diff --git a/dev_scripts_helpers/release_sorrentum/filter_repo/lint_history.py b/dev_scripts_helpers/release_sorrentum/filter_repo/lint_history.py index 65bf6247a..b4431629d 100755 --- a/dev_scripts_helpers/release_sorrentum/filter_repo/lint_history.py +++ b/dev_scripts_helpers/release_sorrentum/filter_repo/lint_history.py @@ -145,8 +145,12 @@ def lint_with_real_filenames(commit, metadata): # Get the old blob contents cat_file_process.stdin.write(change.blob_id + b"\n") cat_file_process.stdin.flush() - objhash, objtype, objsize = cat_file_process.stdout.readline().split() - contents_plus_newline = cat_file_process.stdout.read(int(objsize) + 1) + objhash, objtype, objsize = ( + cat_file_process.stdout.readline().split() + ) + contents_plus_newline = cat_file_process.stdout.read( + int(objsize) + 1 + ) # Write it out to a file with the same basename filename = os.path.join(tmpdir, os.path.basename(change.filename)) @@ -170,7 +174,7 @@ def lint_with_real_filenames(commit, metadata): def lint_non_binary_blobs(blob, metadata): - if not b"\0" in blob.data[0:8192]: + if b"\0" not in blob.data[0:8192]: filename = ".git/info/tmpfile" with open(filename, "wb") as f: f.write(blob.data) diff --git a/dev_scripts_helpers/scraping_script/SorrTask396_scraping_script.ipynb b/dev_scripts_helpers/scraping_script/SorrTask396_scraping_script.ipynb index ac7c8bc4d..4f1b60910 100644 --- a/dev_scripts_helpers/scraping_script/SorrTask396_scraping_script.ipynb +++ b/dev_scripts_helpers/scraping_script/SorrTask396_scraping_script.ipynb @@ -22,7 +22,7 @@ "url = \"https://www.cs.umd.edu/people/phonebook/grad-student\"\n", "response = requests.get(url)\n", "\n", - "soup = BeautifulSoup(response.text, 'html.parser')" + "soup = BeautifulSoup(response.text, \"html.parser\")" ] }, { @@ -33,17 +33,21 @@ "outputs": [], "source": [ "def _extract_contact(student):\n", - " contact = student.find('td', {'class': 'views-field-contact'})\n", - " contact_link = contact.find('a')\n", + " contact = student.find(\"td\", {\"class\": \"views-field-contact\"})\n", + " contact_link = contact.find(\"a\")\n", " if contact_link:\n", - " contact_link = contact_link.get('href') if contact_link else \"\"\n", - " email = 'https://www.cs.umd.edu' + contact_link\n", + " contact_link = contact_link.get(\"href\") if contact_link else \"\"\n", + " email = \"https://www.cs.umd.edu\" + contact_link\n", " else:\n", - " user_element = contact.find('span', {'class': 'u'})\n", + " user_element = contact.find(\"span\", {\"class\": \"u\"})\n", " user = user_element.text if user_element else \"\"\n", - " domain_element = contact.find('span', {'class': 'd'})\n", - " domain = domain_element.text.replace(' ', '') if domain_element else \"\"\n", - " email = user + \"@\" + domain.replace('[.dot.]', '.') if user and domain else \"\"\n", + " domain_element = contact.find(\"span\", {\"class\": \"d\"})\n", + " domain = domain_element.text.replace(\" \", \"\") if domain_element else \"\"\n", + " email = (\n", + " user + \"@\" + domain.replace(\"[.dot.]\", \".\")\n", + " if user and domain\n", + " else \"\"\n", + " )\n", " return email" ] }, @@ -55,8 +59,10 @@ "outputs": [], "source": [ "def _extract_website(student):\n", - " website = student.find('td', {'class': 'views-field-field-profile-website'}).find('a')\n", - " website_link = website.get('href') if website else \"\"\n", + " website = student.find(\n", + " \"td\", {\"class\": \"views-field-field-profile-website\"}\n", + " ).find(\"a\")\n", + " website_link = website.get(\"href\") if website else \"\"\n", " return website_link" ] }, @@ -68,8 +74,10 @@ "outputs": [], "source": [ "def _extract_photo_link(student):\n", - " photo = student.find('td', {'class': 'views-field views-field-field-person-photo'})\n", - " photo_link = photo.find('img').get('src')\n", + " photo = student.find(\n", + " \"td\", {\"class\": \"views-field views-field-field-person-photo\"}\n", + " )\n", + " photo_link = photo.find(\"img\").get(\"src\")\n", " return photo_link" ] }, @@ -81,18 +89,51 @@ "outputs": [], "source": [ "data = []\n", - "for student in soup.select('tr.odd, tr.even'):\n", - " last_name = student.find('td', {'class': 'views-field-field-person-last-name'}).text.strip()\n", - " first_name = student.find('td', {'class': 'views-field-field-person-first-name'}).text.strip()\n", - " degree = student.find('td', {'class': 'views-field-field-degree'}).text.strip()\n", - " location = student.find('td', {'class': 'views-field-field-profile-location'}).text.strip()\n", - " phone = student.find('td', {'class': 'views-field-field-profile-phone'}).text.strip()\n", + "for student in soup.select(\"tr.odd, tr.even\"):\n", + " last_name = student.find(\n", + " \"td\", {\"class\": \"views-field-field-person-last-name\"}\n", + " ).text.strip()\n", + " first_name = student.find(\n", + " \"td\", {\"class\": \"views-field-field-person-first-name\"}\n", + " ).text.strip()\n", + " degree = student.find(\n", + " \"td\", {\"class\": \"views-field-field-degree\"}\n", + " ).text.strip()\n", + " location = student.find(\n", + " \"td\", {\"class\": \"views-field-field-profile-location\"}\n", + " ).text.strip()\n", + " phone = student.find(\n", + " \"td\", {\"class\": \"views-field-field-profile-phone\"}\n", + " ).text.strip()\n", " email = _extract_contact(student)\n", " website = _extract_website(student)\n", " photo_link = _extract_photo_link(student)\n", - " data.append([last_name, first_name, degree, location, phone, email, website, photo_link])\n", + " data.append(\n", + " [\n", + " last_name,\n", + " first_name,\n", + " degree,\n", + " location,\n", + " phone,\n", + " email,\n", + " website,\n", + " photo_link,\n", + " ]\n", + " )\n", "\n", - "df = pd.DataFrame(data, columns=['Last Name', 'First Name', 'Degree', 'Location', 'Phone', 'Email', 'Website', 'Photo_link'])" + "df = pd.DataFrame(\n", + " data,\n", + " columns=[\n", + " \"Last Name\",\n", + " \"First Name\",\n", + " \"Degree\",\n", + " \"Location\",\n", + " \"Phone\",\n", + " \"Email\",\n", + " \"Website\",\n", + " \"Photo_link\",\n", + " ],\n", + ")" ] }, { @@ -1125,7 +1166,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.to_csv('UMD_CS_Phonebook_Graduate_Students.csv')" + "df.to_csv(\"UMD_CS_Phonebook_Graduate_Students.csv\")" ] } ], diff --git a/dev_scripts_helpers/system_tools/capture_notebook_cells.py b/dev_scripts_helpers/system_tools/capture_notebook_cells.py index 4424f537f..1c239ba5f 100644 --- a/dev_scripts_helpers/system_tools/capture_notebook_cells.py +++ b/dev_scripts_helpers/system_tools/capture_notebook_cells.py @@ -1,14 +1,14 @@ # pip install nbconvert nbformat selenium pillow -#import os -#import time +# import os +# import time -#import nbformat -#from PIL import Image -#from selenium import webdriver +# import nbformat +# from PIL import Image +# from selenium import webdriver -#def extract_cells(notebook_path, cell_numbers, temp_notebook="temp_subset.ipynb"): +# def extract_cells(notebook_path, cell_numbers, temp_notebook="temp_subset.ipynb"): # """ # Extracts specific cells from a Jupyter Notebook and saves them as a new # temporary notebook. @@ -30,7 +30,7 @@ # nbformat.write(nb, f) # # -#def save_cells_as_png(notebook_path, cell_numbers, output_path="output.png"): +# def save_cells_as_png(notebook_path, cell_numbers, output_path="output.png"): # """ # Extracts specific cells, renders them as HTML, and captures a screenshot. # @@ -75,8 +75,8 @@ # # ## Example: Capture cells 1, 3, and 5 from notebook -#save_cells_as_png( +# save_cells_as_png( # "example_notebook.ipynb", # cell_numbers=[1, 3, 5], # output_path="selected_cells.png", -#) +# ) diff --git a/dev_scripts_helpers/system_tools/extract_cfile.py b/dev_scripts_helpers/system_tools/extract_cfile.py index 9d52dfc21..8b59c4819 100755 --- a/dev_scripts_helpers/system_tools/extract_cfile.py +++ b/dev_scripts_helpers/system_tools/extract_cfile.py @@ -22,7 +22,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_input_output_args(parser) hparser.add_verbosity_arg(parser) diff --git a/dev_scripts_helpers/system_tools/ffind.py b/dev_scripts_helpers/system_tools/ffind.py index 03819ad6a..25d80027c 100755 --- a/dev_scripts_helpers/system_tools/ffind.py +++ b/dev_scripts_helpers/system_tools/ffind.py @@ -34,7 +34,8 @@ def _print_help(parser): def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "positional", diff --git a/dev_scripts_helpers/system_tools/jackdoc.py b/dev_scripts_helpers/system_tools/jackdoc.py index 68fde282d..4e120ba01 100644 --- a/dev_scripts_helpers/system_tools/jackdoc.py +++ b/dev_scripts_helpers/system_tools/jackdoc.py @@ -5,6 +5,7 @@ Example usage: jackdoc "search_term" [--skip-toc] [--line-only] [--subdir ] """ + import argparse import logging import os @@ -185,7 +186,8 @@ def _parse() -> argparse.ArgumentParser: :return: the argument parser object """ parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("search_term", help="Term to search in Markdown files") parser.add_argument( diff --git a/dev_scripts_helpers/system_tools/remove_escape_chars.py b/dev_scripts_helpers/system_tools/remove_escape_chars.py index f89e1562e..29f1ebb64 100755 --- a/dev_scripts_helpers/system_tools/remove_escape_chars.py +++ b/dev_scripts_helpers/system_tools/remove_escape_chars.py @@ -20,7 +20,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_input_output_args(parser) hparser.add_verbosity_arg(parser) diff --git a/dev_scripts_helpers/system_tools/replace_text.py b/dev_scripts_helpers/system_tools/replace_text.py index dbce24648..e19b446ae 100755 --- a/dev_scripts_helpers/system_tools/replace_text.py +++ b/dev_scripts_helpers/system_tools/replace_text.py @@ -29,7 +29,9 @@ _ENCODING = "utf-8" -def _get_all_files(dirs: List[str], extensions: Optional[List[str]]) -> List[str]: +def _get_all_files( + dirs: List[str], extensions: Optional[List[str]] +) -> List[str]: """ Find all the files with the given extensions in files under `dirs`. @@ -71,8 +73,9 @@ def _get_all_files(dirs: List[str], extensions: Optional[List[str]]) -> List[str d, pattern, only_files, use_relative_paths ) _LOG.debug( - "extensions=%s -> found %s files", extensions, - len(file_names_tmp) + "extensions=%s -> found %s files", + extensions, + len(file_names_tmp), ) file_names.extend(file_names_tmp) # Exclude some files. @@ -147,7 +150,9 @@ def _get_files_to_replace( file_names_to_process.append(f) # txt = "\n".join(res) - _LOG.info("Found %s occurrences to replace\n%s", len(res), hprint.indent(txt)) + _LOG.info( + "Found %s occurrences to replace\n%s", len(res), hprint.indent(txt) + ) _LOG.info("Found %s files to replace", len(file_names_to_process)) return file_names_to_process, txt @@ -369,7 +374,10 @@ def _prerelease_cleanup(args: argparse.Namespace) -> None: Implement AmpTask1403. """ # From longest to shortest to avoid nested replacements. - to_replace = [("instrument_master", "im"), ("order_management_system", "oms")] + to_replace = [ + ("instrument_master", "im"), + ("order_management_system", "oms"), + ] dirs = ["."] # exts = ["py", "ipynb", "md", "txt"] exts = None @@ -689,7 +697,9 @@ def _main(parser: argparse.ArgumentParser) -> None: ) if args.preview: hio.to_file("./cfile", txt) - _LOG.warning("Preview only as required. Results saved in ./cfile") + _LOG.warning( + "Preview only as required. Results saved in ./cfile" + ) else: # Replace the string inside the files. The `args.old` regex is # replaced with the `args.new` regex in the whole files' contents. diff --git a/dev_scripts_helpers/system_tools/save_screenshot.py b/dev_scripts_helpers/system_tools/save_screenshot.py index 68b7e4b4e..d77b291e3 100755 --- a/dev_scripts_helpers/system_tools/save_screenshot.py +++ b/dev_scripts_helpers/system_tools/save_screenshot.py @@ -24,10 +24,13 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("positional", nargs="*", help="...") - parser.add_argument("--dst_dir", action="store", help="Destination directory") + parser.add_argument( + "--dst_dir", action="store", help="Destination directory" + ) parser.add_argument("--filename", action="store", help="File name") parser.add_argument( "--override", action="store_true", help="Override if file exists" diff --git a/dev_scripts_helpers/system_tools/tg.py b/dev_scripts_helpers/system_tools/tg.py index a2083e62f..90bb3e4a8 100755 --- a/dev_scripts_helpers/system_tools/tg.py +++ b/dev_scripts_helpers/system_tools/tg.py @@ -28,7 +28,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("-m", "--msg", action="store", default="done", type=str) parser.add_argument( diff --git a/dev_scripts_helpers/system_tools/zip_files.py b/dev_scripts_helpers/system_tools/zip_files.py index 95816d65a..bf4a4bc8a 100755 --- a/dev_scripts_helpers/system_tools/zip_files.py +++ b/dev_scripts_helpers/system_tools/zip_files.py @@ -34,7 +34,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--src_dir", action="store", required=True, help="Source dir" diff --git a/dev_scripts_helpers/test/test_find_unused_golden_files.py b/dev_scripts_helpers/test/test_find_unused_golden_files.py index f87f405ea..68c4db145 100644 --- a/dev_scripts_helpers/test/test_find_unused_golden_files.py +++ b/dev_scripts_helpers/test/test_find_unused_golden_files.py @@ -21,8 +21,8 @@ def test2(self) -> None: self.assertEqual(actual, expected) """ test_py_file = "dir/test/test_lib.py" - actual_test_methods, actual_class_to_test_file = dsfugofi.parse_test_code( - code, test_py_file + actual_test_methods, actual_class_to_test_file = ( + dsfugofi.parse_test_code(code, test_py_file) ) expected_test_methods = ["dir/test/outcomes/TestClass.test1"] expected_class_to_test_file = { @@ -42,8 +42,8 @@ def test1(self) -> None: self.helper(actual) """ test_py_file = "dir/test/test_lib.py" - actual_test_methods, actual_class_to_test_file = dsfugofi.parse_test_code( - code, test_py_file + actual_test_methods, actual_class_to_test_file = ( + dsfugofi.parse_test_code(code, test_py_file) ) expected_test_methods = ["dir/test/outcomes/TestClass.test1"] expected_class_to_test_file = { @@ -64,8 +64,8 @@ def test1(self) -> None: self.helper(actual) """ test_py_file = "dir/test/test_lib.py" - actual_test_methods, actual_class_to_test_file = dsfugofi.parse_test_code( - code, test_py_file + actual_test_methods, actual_class_to_test_file = ( + dsfugofi.parse_test_code(code, test_py_file) ) expected_test_methods: List[str] = [] expected_class_to_test_file: Dict[str, str] = { diff --git a/dev_scripts_helpers/test/test_process_jupytext.py b/dev_scripts_helpers/test/test_process_jupytext.py index 8623864aa..09c7dfea2 100644 --- a/dev_scripts_helpers/test/test_process_jupytext.py +++ b/dev_scripts_helpers/test/test_process_jupytext.py @@ -18,7 +18,6 @@ class Test_process_jupytext(hunitest.TestCase): - @pytest.mark.slow("~7 seconds.") def test_end_to_end(self) -> None: """ diff --git a/dev_scripts_helpers/test/test_publish_notebook.py b/dev_scripts_helpers/test/test_publish_notebook.py index f93642b5d..cc8cc4d30 100644 --- a/dev_scripts_helpers/test/test_publish_notebook.py +++ b/dev_scripts_helpers/test/test_publish_notebook.py @@ -8,6 +8,7 @@ _LOG = logging.getLogger(__name__) + @pytest.mark.skip(reason="TODO(Juraj): HelpersTask21.") class Test_publish_notebook1(hunitest.TestCase): @pytest.mark.slow("~7 seconds, see CmTask4951.") diff --git a/dev_scripts_helpers/test/test_replace_text.py b/dev_scripts_helpers/test/test_replace_text.py index e04b0b309..f8e645584 100644 --- a/dev_scripts_helpers/test/test_replace_text.py +++ b/dev_scripts_helpers/test/test_replace_text.py @@ -130,7 +130,10 @@ def test4(self) -> None: replace_in = "basename" filter_by = r"test\d" filter_on = "basename" - expected_file_names = ["dir/Test.subdir/test2.py", "dir/subdir/test1.py"] + expected_file_names = [ + "dir/Test.subdir/test2.py", + "dir/subdir/test1.py", + ] expected_file_map = [ ("dir/Test.subdir/test2.py", "dir/Test.subdir/testNUM.py"), ("dir/subdir/test1.py", "dir/subdir/testNUM.py"), @@ -157,7 +160,10 @@ def test5(self) -> None: replace_in = "dirname" filter_by = r"test\d" filter_on = "basename" - expected_file_names = ["dir/Test.subdir/test2.py", "dir/subdir/test1.py"] + expected_file_names = [ + "dir/Test.subdir/test2.py", + "dir/subdir/test1.py", + ] expected_file_map = [ ("dir/Test.subdir/test2.py", "dir/outcomes/Test.subdir/test2.py"), ("dir/subdir/test1.py", "dir/outcomes/subdir/test1.py"), diff --git a/dev_scripts_helpers/test/test_run_notebook.py b/dev_scripts_helpers/test/test_run_notebook.py index 9d571d3b4..4ecb44755 100644 --- a/dev_scripts_helpers/test/test_run_notebook.py +++ b/dev_scripts_helpers/test/test_run_notebook.py @@ -180,7 +180,9 @@ def run_cmd_line( "expected_pass=%s abort_on_error=%s", expected_pass, abort_on_error ) _LOG.debug("cmd=%s", cmd) - rc = hsystem.system(cmd, abort_on_error=abort_on_error, suppress_output=False) + rc = hsystem.system( + cmd, abort_on_error=abort_on_error, suppress_output=False + ) if expected_pass: self.assertEqual(rc, 0) else: diff --git a/dev_scripts_helpers/test/test_toml_merge.py b/dev_scripts_helpers/test/test_toml_merge.py index d1d389a1c..6c33eee16 100644 --- a/dev_scripts_helpers/test/test_toml_merge.py +++ b/dev_scripts_helpers/test/test_toml_merge.py @@ -9,15 +9,15 @@ import dev_scripts_helpers.coding_tools.toml_merge as dsctomer import helpers.hunit_test as hunitest - def _to_toml(txt: str) -> MutableMapping[str, Any]: """ Remove all empty lines and leading / trailing spaces. """ - txt = "\n".join([line.rstrip().lstrip() for line in txt.split("\n") if txt]) + txt = "\n".join( + [line.rstrip().lstrip() for line in txt.split("\n") if txt] + ) return toml.load(StringIO(txt)) - class TestMergeToml(hunitest.TestCase): def test1(self) -> None: """ diff --git a/dev_scripts_helpers/thin_client/build.py b/dev_scripts_helpers/thin_client/build.py index 68f7a25e4..96c5f7a0c 100755 --- a/dev_scripts_helpers/thin_client/build.py +++ b/dev_scripts_helpers/thin_client/build.py @@ -69,7 +69,9 @@ def _main(parser: argparse.ArgumentParser) -> None: # Install the requirements. thin_environ_dir = tcu.get_thin_environment_dir(dir_suffix) requirements_path = os.path.join(thin_environ_dir, "requirements.txt") - tmp_requirements_path = os.path.join(thin_environ_dir, "tmp.requirements.txt") + tmp_requirements_path = os.path.join( + thin_environ_dir, "tmp.requirements.txt" + ) shutil.copy(requirements_path, tmp_requirements_path) if platform.system() == "Darwin" or ( platform.system() == "Linux" and not hserver.is_dev_csfy() @@ -126,7 +128,8 @@ def _main(parser: argparse.ArgumentParser) -> None: def _parse() -> argparse.ArgumentParser: # Create the parser. parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) hparser.add_verbosity_arg(parser) parser.add_argument( diff --git a/dev_scripts_helpers/to_clean/gen_utils.ORIG.py b/dev_scripts_helpers/to_clean/gen_utils.ORIG.py index 7e7c26a48..265b86713 100644 --- a/dev_scripts_helpers/to_clean/gen_utils.ORIG.py +++ b/dev_scripts_helpers/to_clean/gen_utils.ORIG.py @@ -755,7 +755,9 @@ def remove_outliers( if mode is None: mode = "winsorize" log.debug("Removing outliers with mode=%s", mode) - bounds = utils.jstats.get_quantile_bounds(obj, lower_quantile, upper_quantile) + bounds = utils.jstats.get_quantile_bounds( + obj, lower_quantile, upper_quantile + ) if print_stats: log.debug("bounds=%s", str(bounds)) if inplace: @@ -768,14 +770,17 @@ def remove_outliers( if print_stats: num = np.sum(obj <= bounds[0]) + np.sum(bounds[1] <= obj) log.debug( - "winsorize: to_process=%s", dbg.perc(num, len(ret), printAll=True) + "winsorize: to_process=%s", + dbg.perc(num, len(ret), printAll=True), ) else: mask = (bounds[0] <= obj) & (obj <= bounds[1]) if print_stats: num = np.sum(mask) log.debug( - "%s: to_process=%s", mode, dbg.perc(num, len(ret), printAll=True) + "%s: to_process=%s", + mode, + dbg.perc(num, len(ret), printAll=True), ) if mode == "set_to_nan": ret[~mask] = np.nan @@ -816,7 +821,10 @@ def remove_outlier_rows_from_df( if col in col_names_to_trim: log.debug("Trimming col %s", col) trimmed_col, _ = remove_outliers( - df[col], lower_quantile, upper_quantile=upper_quantile, mode=mode + df[col], + lower_quantile, + upper_quantile=upper_quantile, + mode=mode, ) else: log.debug("Skipping col %s", col) @@ -926,7 +934,8 @@ def report_intraday_stats(rets): # stats_df.append(row) stats_df = pd.DataFrame( - stats_df, columns=["inst", "min_hour", "max_hour", "min_date", "max_date"] + stats_df, + columns=["inst", "min_hour", "max_hour", "min_date", "max_date"], ) stats_df.set_index("inst", drop=True, inplace=True) return stats_df @@ -938,9 +947,7 @@ def plot_intraday_stats(rets): for i, inst_name in enumerate(inst_names): ax = plt.subplot(len(inst_names) + 1, 1, i + 1) rets_tmp = rets[inst_name].astype(float) - rets_tmp.groupby( - lambda x: x.time() - ).count().plot(ax=ax, title=inst_name) + rets_tmp.groupby(lambda x: x.time()).count().plot(ax=ax, title=inst_name) plt.plot() @@ -1445,7 +1452,9 @@ def compute_correlation( y = (y - y.mean()) / y.std() rho, p_val = scipy.stats.stats.pearsonr(x, y) if print_stats: - print("num_samples=%s" % dbg.perc(len(x), tot_num_samples, printAll=True)) + print( + "num_samples=%s" % dbg.perc(len(x), tot_num_samples, printAll=True) + ) print("rho=%.4f" % rho) print( "2-tailed pvalue=%.4f (%s)" @@ -1567,9 +1576,7 @@ def regress( size=jointplot_size, ) else: - log.warning( - "Skipping plots since there are too many " "predictors" - ) + log.warning("Skipping plots since there are too many predictors") # Robust regression. if robust_regress: # From http://scikit-learn.org/stable/auto_examples/linear_model/plot_robust_fit.html#sphx-glr-auto-examples-linear-model-plot-robust-fit-py diff --git a/dev_scripts_helpers/to_clean/gen_utils.py b/dev_scripts_helpers/to_clean/gen_utils.py index 1d7fc1c42..e0fab931a 100644 --- a/dev_scripts_helpers/to_clean/gen_utils.py +++ b/dev_scripts_helpers/to_clean/gen_utils.py @@ -732,7 +732,9 @@ def remove_outliers( if mode is None: mode = "winsorize" _LOG.debug("Removing outliers with mode=%s", mode) - bounds = utils.jstats.get_quantile_bounds(obj, lower_quantile, upper_quantile) + bounds = utils.jstats.get_quantile_bounds( + obj, lower_quantile, upper_quantile + ) if print_stats: _LOG.debug("bounds=%s", str(bounds)) if inplace: @@ -745,14 +747,17 @@ def remove_outliers( if print_stats: num = np.sum(obj <= bounds[0]) + np.sum(bounds[1] <= obj) _LOG.debug( - "winsorize: to_process=%s", dbg.perc(num, len(ret), printAll=True) + "winsorize: to_process=%s", + dbg.perc(num, len(ret), printAll=True), ) else: mask = (bounds[0] <= obj) & (obj <= bounds[1]) if print_stats: num = np.sum(mask) _LOG.debug( - "%s: to_process=%s", mode, dbg.perc(num, len(ret), printAll=True) + "%s: to_process=%s", + mode, + dbg.perc(num, len(ret), printAll=True), ) if mode == "set_to_nan": ret[~mask] = np.nan @@ -793,7 +798,10 @@ def remove_outlier_rows_from_df( if col in col_names_to_trim: _LOG.debug("Trimming col %s", col) trimmed_col, _ = remove_outliers( - df[col], lower_quantile, upper_quantile=upper_quantile, mode=mode + df[col], + lower_quantile, + upper_quantile=upper_quantile, + mode=mode, ) else: _LOG.debug("Skipping col %s", col) @@ -903,7 +911,8 @@ def report_intraday_stats(rets): # stats_df.append(row) stats_df = pd.DataFrame( - stats_df, columns=["inst", "min_hour", "max_hour", "min_date", "max_date"] + stats_df, + columns=["inst", "min_hour", "max_hour", "min_date", "max_date"], ) stats_df.set_index("inst", drop=True, inplace=True) return stats_df @@ -1420,7 +1429,9 @@ def compute_correlation( y = (y - y.mean()) / y.std() rho, p_val = scipy.stats.stats.pearsonr(x, y) if print_stats: - print("num_samples=%s" % dbg.perc(len(x), tot_num_samples, printAll=True)) + print( + "num_samples=%s" % dbg.perc(len(x), tot_num_samples, printAll=True) + ) print("rho=%.4f" % rho) print( "2-tailed pvalue=%.4f (%s)" diff --git a/dev_scripts_helpers/update_devops_packages/notebooks/CmampTask7292_ns_vs_us.ipynb b/dev_scripts_helpers/update_devops_packages/notebooks/CmampTask7292_ns_vs_us.ipynb index edfbb7e1f..80f6bc2d9 100644 --- a/dev_scripts_helpers/update_devops_packages/notebooks/CmampTask7292_ns_vs_us.ipynb +++ b/dev_scripts_helpers/update_devops_packages/notebooks/CmampTask7292_ns_vs_us.ipynb @@ -381,7 +381,11 @@ "outputs": [], "source": [ "def test_write_and_read_parquet_file_with_unit(\n", - " initial_df: pd.DataFrame, file_name: str, unit: str, *, clean_up: bool = False\n", + " initial_df: pd.DataFrame,\n", + " file_name: str,\n", + " unit: str,\n", + " *,\n", + " clean_up: bool = False,\n", ") -> None:\n", " \"\"\"\n", " Write the provided DataFrame to Parquet file and read it back, verifying\n", diff --git a/dev_scripts_helpers/update_devops_packages/notebooks/CmampTask7292_ns_vs_us.py b/dev_scripts_helpers/update_devops_packages/notebooks/CmampTask7292_ns_vs_us.py index 5169e1a21..886b94e82 100644 --- a/dev_scripts_helpers/update_devops_packages/notebooks/CmampTask7292_ns_vs_us.py +++ b/dev_scripts_helpers/update_devops_packages/notebooks/CmampTask7292_ns_vs_us.py @@ -118,7 +118,11 @@ def test_write_and_read_partition_parquet_with_unit( # %% def test_write_and_read_parquet_file_with_unit( - initial_df: pd.DataFrame, file_name: str, unit: str, *, clean_up: bool = False + initial_df: pd.DataFrame, + file_name: str, + unit: str, + *, + clean_up: bool = False, ) -> None: """ Write the provided DataFrame to Parquet file and read it back, verifying diff --git a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb index 48f2b4c28..4ca46c5a7 100644 --- a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb +++ b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb @@ -1,377 +1,378 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "CONTENTS:\n", - "- [Description](#description)\n", - "- [Imports](#imports)\n", - "- [Utils](#utils)\n", - "- [GH workflows state](#gh-workflows-state)\n", - "- [Allure reports](#allure-reports)\n", - "- [Number of open pull requests](#number-of-open-pull-requests)\n", - "- [Code coverage HTML-page](#code-coverage-html-page)\n", - "- [Code Coverage Page - CodeCov](#code-coverage-page---codecov)" - ] - }, - { - "cell_type": "markdown", - "id": "e3103ff3", - "metadata": {}, - "source": [ - " TODO(Grisha): does it belong to the `devops` dir?" - ] - }, - { - "cell_type": "markdown", - "id": "9d992fed", - "metadata": {}, - "source": [ - "\n", - "# Description" - ] - }, - { - "cell_type": "markdown", - "id": "3e381a7d", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-02T08:02:05.889049Z", - "start_time": "2024-02-02T08:02:05.883420Z" - } - }, - "source": [ - "The notebook reports the latest build status for multiple repos." - ] - }, - { - "cell_type": "markdown", - "id": "982f47f1", - "metadata": {}, - "source": [ - "\n", - "# Imports" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "97bbec36", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:42.038091Z", - "start_time": "2024-02-07T17:59:42.002068Z" - } - }, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "518df056", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:42.078514Z", - "start_time": "2024-02-07T17:59:42.041301Z" - } - }, - "outputs": [], - "source": [ - "import logging\n", - "from typing import Dict\n", - "\n", - "import pandas as pd\n", - "from IPython.display import Markdown, display\n", - "\n", - "import helpers.hdbg as hdbg\n", - "import helpers.henv as henv\n", - "import helpers.hpandas as hpandas\n", - "import helpers.hprint as hprint\n", - "import helpers.lib_tasks_gh as hlitagh\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f0793aa5", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:42.268049Z", - "start_time": "2024-02-07T17:59:42.081426Z" - } - }, - "outputs": [], - "source": [ - "hdbg.init_logger(verbosity=logging.INFO)\n", - "_LOG = logging.getLogger(__name__)\n", - "_LOG.info(\"%s\", henv.get_system_signature()[0])\n", - "hprint.config_notebook()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "93c2d39f", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:42.338614Z", - "start_time": "2024-02-07T17:59:42.271472Z" - } - }, - "outputs": [], - "source": [ - "# Set the display options to print the full table.\n", - "pd.set_option(\"display.max_colwidth\", None)\n", - "pd.set_option(\"display.max_columns\", None)" - ] - }, - { - "cell_type": "markdown", - "id": "14f379d5", - "metadata": { - "lines_to_next_cell": 2 - }, - "source": [ - "\n", - "# Utils" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1f41a8dd", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:42.380319Z", - "start_time": "2024-02-07T17:59:42.343492Z" - } - }, - "outputs": [], - "source": [ - "def make_clickable(url: str) -> str:\n", - " \"\"\"\n", - " Wrapper to make the URL value clickable.\n", - "\n", - " :param url: URL value to convert\n", - " :return: clickable URL link\n", - " \"\"\"\n", - " return f'{url}'\n", - "\n", - "\n", - "def color_format(val: str, status_color_mapping: Dict[str, str]) -> str:\n", - " \"\"\"\n", - " Return the color depends on status.\n", - "\n", - " :param val: value of the status e.g. `failure`\n", - " :param status_color_mapping: mapping statuses to the colors e.g.:\n", - " ```\n", - " {\n", - " \"success\": \"green\",\n", - " \"failure\": \"red\",\n", - " }\n", - " ```\n", - " \"\"\"\n", - " if val in status_color_mapping:\n", - " color = status_color_mapping[val]\n", - " else:\n", - " color = \"grey\"\n", - " return f\"background-color: {color}\"" - ] - }, - { - "cell_type": "markdown", - "id": "189f2c75", - "metadata": {}, - "source": [ - "\n", - "# GH workflows state" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "865bc9f2", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:57.513155Z", - "start_time": "2024-02-07T17:59:42.383039Z" - } - }, - "outputs": [], - "source": [ - "repo_list = [\n", - " \"cryptokaizen/cmamp\",\n", - " \"cryptokaizen/orange\",\n", - " \"cryptokaizen/lemonade\",\n", - " \"causify-ai/kaizenflow\",\n", - "]\n", - "workflow_df = hlitagh.gh_get_details_for_all_workflows(repo_list)\n", - "# Reorder columns.\n", - "columns_order = [\"repo_name\", \"workflow_name\", \"conclusion\", \"url\"]\n", - "workflow_df = workflow_df[columns_order]\n", - "# Make URL values clickable.\n", - "workflow_df[\"url\"] = workflow_df[\"url\"].apply(make_clickable)\n", - "_LOG.info(hpandas.df_to_str(workflow_df, log_level=logging.INFO))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f7e999ce", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:57.585606Z", - "start_time": "2024-02-07T17:59:57.515915Z" - } - }, - "outputs": [], - "source": [ - "status_color_mapping = {\n", - " \"success\": \"green\",\n", - " \"failure\": \"red\",\n", - "}\n", - "repos = workflow_df[\"repo_name\"].unique()\n", - "display(Markdown(\"## Overall Status\"))\n", - "current_timestamp = pd.Timestamp.now(tz=\"America/New_York\")\n", - "display(Markdown(f\"**Last run: {current_timestamp}**\"))\n", - "for repo in repos:\n", - " # Calculate the overall status.\n", - " repo_df = workflow_df[workflow_df[\"repo_name\"] == repo]\n", - " overall_status = hlitagh.gh_get_overall_build_status_for_repo(repo_df)\n", - " display(Markdown(f\"## {repo}: {overall_status}\"))\n", - " repo_df = repo_df.drop(columns=[\"repo_name\"])\n", - " display(\n", - " repo_df.style.map(\n", - " color_format,\n", - " status_color_mapping=status_color_mapping,\n", - " subset=[\"conclusion\"],\n", - " )\n", - " )" - ] - }, - { - "cell_type": "markdown", - "id": "a00870a9", - "metadata": {}, - "source": [ - "\n", - "# Allure reports" - ] - }, - { - "cell_type": "markdown", - "id": "36e93fca", - "metadata": {}, - "source": [ - "- fast tests: http://172.30.2.44/allure_reports/cmamp/fast/latest/index.html\n", - "- slow tests: http://172.30.2.44/allure_reports/cmamp/slow/latest/index.html\n", - "- superslow tests: http://172.30.2.44/allure_reports/cmamp/superslow/latest/index.html" - ] - }, - { - "cell_type": "markdown", - "id": "bb8ed505", - "metadata": {}, - "source": [ - "\n", - "# Number of open pull requests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "69dbda1d", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:59.309022Z", - "start_time": "2024-02-07T17:59:57.588291Z" - } - }, - "outputs": [], - "source": [ - "for repo in repo_list:\n", - " number_prs = len(hlitagh.gh_get_open_prs(repo))\n", - " _LOG.info(\"%s: %s\", repo, number_prs)" - ] - }, - { - "cell_type": "markdown", - "id": "ec63cb5e", - "metadata": {}, - "source": [ - "\n", - "# Code coverage HTML-page" - ] - }, - { - "cell_type": "markdown", - "id": "569f9404", - "metadata": { - "ExecuteTime": { - "end_time": "2024-01-24T14:40:31.379819Z", - "start_time": "2024-01-24T14:40:31.327151Z" - } - }, - "source": [ - "http://172.30.2.44/html_coverage/runner_master/" - ] - }, - { - "cell_type": "markdown", - "id": "027d1b3d", - "metadata": {}, - "source": [ - "\n", - "# Code Coverage Page - CodeCov" - ] - }, - { - "cell_type": "markdown", - "id": "6c9219e1", - "metadata": {}, - "source": [ - "- Helpers: https://app.codecov.io/gh/causify-ai/helpers" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "id": "7fb27b941602401d91542211134fc71a", + "metadata": {}, + "source": [ + "CONTENTS:\n", + "- [Description](#description)\n", + "- [Imports](#imports)\n", + "- [Utils](#utils)\n", + "- [GH workflows state](#gh-workflows-state)\n", + "- [Allure reports](#allure-reports)\n", + "- [Number of open pull requests](#number-of-open-pull-requests)\n", + "- [Code coverage HTML-page](#code-coverage-html-page)\n", + "- [Code Coverage Page - CodeCov](#code-coverage-page---codecov)" + ] + }, + { + "cell_type": "markdown", + "id": "e3103ff3", + "metadata": {}, + "source": [ + " TODO(Grisha): does it belong to the `devops` dir?" + ] + }, + { + "cell_type": "markdown", + "id": "9d992fed", + "metadata": {}, + "source": [ + "\n", + "# Description" + ] + }, + { + "cell_type": "markdown", + "id": "3e381a7d", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-02T08:02:05.889049Z", + "start_time": "2024-02-02T08:02:05.883420Z" + } + }, + "source": [ + "The notebook reports the latest build status for multiple repos." + ] + }, + { + "cell_type": "markdown", + "id": "982f47f1", + "metadata": {}, + "source": [ + "\n", + "# Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97bbec36", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:42.038091Z", + "start_time": "2024-02-07T17:59:42.002068Z" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "518df056", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:42.078514Z", + "start_time": "2024-02-07T17:59:42.041301Z" + } + }, + "outputs": [], + "source": [ + "import logging\n", + "from typing import Dict\n", + "\n", + "import pandas as pd\n", + "from IPython.display import Markdown, display\n", + "\n", + "import helpers.hdbg as hdbg\n", + "import helpers.henv as henv\n", + "import helpers.hpandas as hpandas\n", + "import helpers.hprint as hprint\n", + "import helpers.lib_tasks_gh as hlitagh" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0793aa5", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:42.268049Z", + "start_time": "2024-02-07T17:59:42.081426Z" + } + }, + "outputs": [], + "source": [ + "hdbg.init_logger(verbosity=logging.INFO)\n", + "_LOG = logging.getLogger(__name__)\n", + "_LOG.info(\"%s\", henv.get_system_signature()[0])\n", + "hprint.config_notebook()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "93c2d39f", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:42.338614Z", + "start_time": "2024-02-07T17:59:42.271472Z" + } + }, + "outputs": [], + "source": [ + "# Set the display options to print the full table.\n", + "pd.set_option(\"display.max_colwidth\", None)\n", + "pd.set_option(\"display.max_columns\", None)" + ] + }, + { + "cell_type": "markdown", + "id": "14f379d5", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "\n", + "# Utils" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f41a8dd", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:42.380319Z", + "start_time": "2024-02-07T17:59:42.343492Z" + } + }, + "outputs": [], + "source": [ + "def make_clickable(url: str) -> str:\n", + " \"\"\"\n", + " Wrapper to make the URL value clickable.\n", + "\n", + " :param url: URL value to convert\n", + " :return: clickable URL link\n", + " \"\"\"\n", + " return f'{url}'\n", + "\n", + "\n", + "def color_format(val: str, status_color_mapping: Dict[str, str]) -> str:\n", + " \"\"\"\n", + " Return the color depends on status.\n", + "\n", + " :param val: value of the status e.g. `failure`\n", + " :param status_color_mapping: mapping statuses to the colors e.g.:\n", + " ```\n", + " {\n", + " \"success\": \"green\",\n", + " \"failure\": \"red\",\n", + " }\n", + " ```\n", + " \"\"\"\n", + " if val in status_color_mapping:\n", + " color = status_color_mapping[val]\n", + " else:\n", + " color = \"grey\"\n", + " return f\"background-color: {color}\"" + ] + }, + { + "cell_type": "markdown", + "id": "189f2c75", + "metadata": {}, + "source": [ + "\n", + "# GH workflows state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "865bc9f2", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:57.513155Z", + "start_time": "2024-02-07T17:59:42.383039Z" + } + }, + "outputs": [], + "source": [ + "repo_list = [\n", + " \"cryptokaizen/cmamp\",\n", + " \"cryptokaizen/orange\",\n", + " \"cryptokaizen/lemonade\",\n", + " \"causify-ai/kaizenflow\",\n", + "]\n", + "workflow_df = hlitagh.gh_get_details_for_all_workflows(repo_list)\n", + "# Reorder columns.\n", + "columns_order = [\"repo_name\", \"workflow_name\", \"conclusion\", \"url\"]\n", + "workflow_df = workflow_df[columns_order]\n", + "# Make URL values clickable.\n", + "workflow_df[\"url\"] = workflow_df[\"url\"].apply(make_clickable)\n", + "_LOG.info(hpandas.df_to_str(workflow_df, log_level=logging.INFO))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f7e999ce", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:57.585606Z", + "start_time": "2024-02-07T17:59:57.515915Z" + } + }, + "outputs": [], + "source": [ + "status_color_mapping = {\n", + " \"success\": \"green\",\n", + " \"failure\": \"red\",\n", + "}\n", + "repos = workflow_df[\"repo_name\"].unique()\n", + "display(Markdown(\"## Overall Status\"))\n", + "current_timestamp = pd.Timestamp.now(tz=\"America/New_York\")\n", + "display(Markdown(f\"**Last run: {current_timestamp}**\"))\n", + "for repo in repos:\n", + " # Calculate the overall status.\n", + " repo_df = workflow_df[workflow_df[\"repo_name\"] == repo]\n", + " overall_status = hlitagh.gh_get_overall_build_status_for_repo(repo_df)\n", + " display(Markdown(f\"## {repo}: {overall_status}\"))\n", + " repo_df = repo_df.drop(columns=[\"repo_name\"])\n", + " display(\n", + " repo_df.style.map(\n", + " color_format,\n", + " status_color_mapping=status_color_mapping,\n", + " subset=[\"conclusion\"],\n", + " )\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "a00870a9", + "metadata": {}, + "source": [ + "\n", + "# Allure reports" + ] + }, + { + "cell_type": "markdown", + "id": "36e93fca", + "metadata": {}, + "source": [ + "- fast tests: http://172.30.2.44/allure_reports/cmamp/fast/latest/index.html\n", + "- slow tests: http://172.30.2.44/allure_reports/cmamp/slow/latest/index.html\n", + "- superslow tests: http://172.30.2.44/allure_reports/cmamp/superslow/latest/index.html" + ] + }, + { + "cell_type": "markdown", + "id": "bb8ed505", + "metadata": {}, + "source": [ + "\n", + "# Number of open pull requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "69dbda1d", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:59.309022Z", + "start_time": "2024-02-07T17:59:57.588291Z" + } + }, + "outputs": [], + "source": [ + "for repo in repo_list:\n", + " number_prs = len(hlitagh.gh_get_open_prs(repo))\n", + " _LOG.info(\"%s: %s\", repo, number_prs)" + ] + }, + { + "cell_type": "markdown", + "id": "ec63cb5e", + "metadata": {}, + "source": [ + "\n", + "# Code coverage HTML-page" + ] + }, + { + "cell_type": "markdown", + "id": "569f9404", + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-24T14:40:31.379819Z", + "start_time": "2024-01-24T14:40:31.327151Z" + } + }, + "source": [ + "http://172.30.2.44/html_coverage/runner_master/" + ] + }, + { + "cell_type": "markdown", + "id": "027d1b3d", + "metadata": {}, + "source": [ + "\n", + "# Code Coverage Page - CodeCov" + ] + }, + { + "cell_type": "markdown", + "id": "6c9219e1", + "metadata": {}, + "source": [ + "- Helpers: https://app.codecov.io/gh/causify-ai/helpers" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/helpers/create_links.py b/helpers/create_links.py index 481b0f152..2e33d11df 100644 --- a/helpers/create_links.py +++ b/helpers/create_links.py @@ -83,10 +83,13 @@ def _parse() -> argparse.ArgumentParser: :return: Argument parser object. """ parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--src_dir", required=True, help="Source directory.") - parser.add_argument("--dst_dir", required=True, help="Destination directory.") + parser.add_argument( + "--dst_dir", required=True, help="Destination directory." + ) parser.add_argument( "--replace_links", action="store_true", @@ -208,7 +211,9 @@ def _replace_with_links( os.remove(dst_file) try: if use_relative_paths: - link_target = os.path.relpath(src_file, os.path.dirname(dst_file)) + link_target = os.path.relpath( + src_file, os.path.dirname(dst_file) + ) else: link_target = os.path.abspath(src_file) os.symlink(link_target, dst_file) diff --git a/helpers/hasyncio.py b/helpers/hasyncio.py index 92d644871..2784e1a4d 100644 --- a/helpers/hasyncio.py +++ b/helpers/hasyncio.py @@ -3,6 +3,7 @@ import helpers.hasyncio as hasynci """ + import asyncio import contextlib import datetime @@ -466,7 +467,9 @@ def sync_wait_until( fractional seconds. """ # Sync wait. - time_in_secs = _wait_until(wait_until_timestamp, get_wall_clock_time, tag=tag) + time_in_secs = _wait_until( + wait_until_timestamp, get_wall_clock_time, tag=tag + ) hdbg.dassert_lte(0, time_in_secs) # TODO(gp): Consider using part of align_on_time_grid for high-precision clock. time.sleep(time_in_secs) @@ -491,7 +494,9 @@ async def async_wait_until( Asynchronous wait until the wall clock time is `timestamp`. """ _LOG.debug(hprint.to_str("wait_until_timestamp")) - time_in_secs = _wait_until(wait_until_timestamp, get_wall_clock_time, tag=tag) + time_in_secs = _wait_until( + wait_until_timestamp, get_wall_clock_time, tag=tag + ) # Async wait. hdbg.dassert_lte(0, time_in_secs) await asyncio.sleep(time_in_secs) diff --git a/helpers/haws.py b/helpers/haws.py index 74dfaae48..0a827bd07 100644 --- a/helpers/haws.py +++ b/helpers/haws.py @@ -144,7 +144,10 @@ def is_task_definition_exists( # TODO(Nikola): Pass a dict config instead, so any part can be updated. def update_task_definition( - task_definition_name: str, new_image_url: str, *, region: Optional[str] = None + task_definition_name: str, + new_image_url: str, + *, + region: Optional[str] = None, ) -> None: """ Create the new revision of specified ECS task definition. diff --git a/helpers/hcache.py b/helpers/hcache.py index d52b377e0..7f9aa54e9 100644 --- a/helpers/hcache.py +++ b/helpers/hcache.py @@ -72,7 +72,9 @@ def enable_clear_cache(val: bool) -> None: global _IS_CLEAR_CACHE_ENABLED if _TRACE: _LOG.trace("") - _LOG.warning("Enabling clear cache to %s -> %s", _IS_CLEAR_CACHE_ENABLED, val) + _LOG.warning( + "Enabling clear cache to %s -> %s", _IS_CLEAR_CACHE_ENABLED, val + ) _IS_CLEAR_CACHE_ENABLED = val @@ -218,7 +220,9 @@ def _create_global_cache_backend( # TODO(gp): -> _get_global_cache -def get_global_cache(cache_type: str, tag: Optional[str] = None) -> joblib.Memory: +def get_global_cache( + cache_type: str, tag: Optional[str] = None +) -> joblib.Memory: """ Get global cache by cache type. @@ -804,9 +808,7 @@ def _get_identifiers( ) # This is needed for joblib >= 1.4.2. func_id = memorized_result.func_id - args_id = memorized_result._get_args_id( - *args, **kwargs - ) + args_id = memorized_result._get_args_id(*args, **kwargs) _LOG.debug("func_id=%s args_id=%s", func_id, args_id) return func_id, args_id diff --git a/helpers/hcache_simple.py b/helpers/hcache_simple.py index 2fa201cc8..c70281dd2 100644 --- a/helpers/hcache_simple.py +++ b/helpers/hcache_simple.py @@ -479,7 +479,6 @@ def reset_cache(func_name: str = "") -> None: def simple_cache( cache_type: str = "json", write_through: bool = False ) -> Callable[..., Any]: - def decorator(func: Callable[..., Any]) -> Callable[..., Any]: hdbg.dassert_in(cache_type, ("json", "pickle")) func_name = func.__name__ @@ -508,7 +507,9 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: hdbg.dassert_in("tot", cache_perf) cache_perf["tot"] += 1 # Handle a forced refresh. - force_refresh = get_cache_property("user", func_name, "force_refresh") + force_refresh = get_cache_property( + "user", func_name, "force_refresh" + ) _LOG.debug("force_refresh=%s", force_refresh) if not force_refresh and key in cache: _LOG.debug("Cache hit for key='%s'", key) diff --git a/helpers/hcfile.py b/helpers/hcfile.py index 8529699f8..8af7eff93 100644 --- a/helpers/hcfile.py +++ b/helpers/hcfile.py @@ -127,4 +127,3 @@ def inject_todos_from_cfile( _LOG.info("Writing %d lines in %s", offset, file_name) txt = "\n".join(txt) hio.to_file(file_name, txt) - diff --git a/helpers/hchatgpt.py b/helpers/hchatgpt.py index ba48dc95d..c257b888b 100644 --- a/helpers/hchatgpt.py +++ b/helpers/hchatgpt.py @@ -475,7 +475,7 @@ def wait_for_run_result(thread_id: str, run_id: str, timeout: int = 180) -> List finished = False _LOG.info("Waiting for chatgpt response...") for i in range(math.ceil(timeout / 5)): - _LOG.info(f"{i*5}/{timeout} seconds before timeout.") + _LOG.info(f"{i * 5}/{timeout} seconds before timeout.") time.sleep(5) run = client.beta.threads.runs.retrieve( thread_id=thread_id, run_id=run_id @@ -527,7 +527,9 @@ def e2e_assistant_runner( thread_id, user_input, input_file_names ) if model: - run_id = run_thread_on_assistant_by_name(assistant_name, thread_id, model) + run_id = run_thread_on_assistant_by_name( + assistant_name, thread_id, model + ) else: run_id = run_thread_on_assistant_by_name(assistant_name, thread_id) messages = wait_for_run_result(thread_id, run_id) diff --git a/helpers/hcsv.py b/helpers/hcsv.py index af792b1b4..b10ac5cd9 100644 --- a/helpers/hcsv.py +++ b/helpers/hcsv.py @@ -337,7 +337,9 @@ def save_csv_as_json( dict_df = convert_csv_to_dict(path_to_csv, remove_nans) # Determine the JSON destination path. if path_to_json is None: - path_to_json = hio.change_filename_extension(path_to_csv, ".csv", ".json") + path_to_json = hio.change_filename_extension( + path_to_csv, ".csv", ".json" + ) # Save the dict into a JSON file. hio.to_json(path_to_json, dict_df) diff --git a/helpers/hdatetime.py b/helpers/hdatetime.py index 78f767d8f..2780e338c 100644 --- a/helpers/hdatetime.py +++ b/helpers/hdatetime.py @@ -277,7 +277,8 @@ def dassert_is_valid_timestamp(timestamp: Optional[pd.Timestamp]) -> None: def dassert_timestamp_lte( - start_timestamp: Optional[pd.Timestamp], end_timestamp: Optional[pd.Timestamp] + start_timestamp: Optional[pd.Timestamp], + end_timestamp: Optional[pd.Timestamp], ) -> None: dassert_is_valid_timestamp(start_timestamp) dassert_is_valid_timestamp(end_timestamp) @@ -286,7 +287,8 @@ def dassert_timestamp_lte( def dassert_timestamp_lt( - start_timestamp: Optional[pd.Timestamp], end_timestamp: Optional[pd.Timestamp] + start_timestamp: Optional[pd.Timestamp], + end_timestamp: Optional[pd.Timestamp], ) -> None: dassert_is_valid_timestamp(start_timestamp) dassert_is_valid_timestamp(end_timestamp) @@ -710,7 +712,7 @@ def shift_to_year_end(x: StrictDatetime) -> StrictDatetime: if match is None: return None span = match.span() - date_without_month = f"{date[:span[0]]}{date[span[1]:]}".strip() + date_without_month = f"{date[: span[0]]}{date[span[1] :]}".strip() if len(date_without_month) == 4 and date_without_month.isdigit(): return shift_to_month_end return None diff --git a/helpers/hdbg.py b/helpers/hdbg.py index b111e9c0c..4362d68b3 100644 --- a/helpers/hdbg.py +++ b/helpers/hdbg.py @@ -151,9 +151,9 @@ def dassert( # Handle the somehow frequent case of using `dassert` instead of another # one, e.g., `dassert(y, list)` if msg is not None: - assert isinstance( - msg, str - ), f"You passed '{msg}' or type '{type(msg)}' instead of str" + assert isinstance(msg, str), ( + f"You passed '{msg}' or type '{type(msg)}' instead of str" + ) if not cond: txt = f"cond={cond}" _dfatal(txt, msg, *args, only_warning=only_warning) @@ -819,7 +819,9 @@ def dassert_dir_exists( def dassert_file_extension( - file_name: str, extensions: Union[str, List[str]], only_warning: bool = False + file_name: str, + extensions: Union[str, List[str]], + only_warning: bool = False, ) -> None: """ Ensure that file has one of the given extensions. diff --git a/helpers/hdict.py b/helpers/hdict.py index 77d8d68d8..f26f25f8c 100644 --- a/helpers/hdict.py +++ b/helpers/hdict.py @@ -6,7 +6,16 @@ import collections import logging -from typing import Any, Dict, Generator, Iterable, Mapping, Optional, Tuple, Union +from typing import ( + Any, + Dict, + Generator, + Iterable, + Mapping, + Optional, + Tuple, + Union, +) import helpers.hdbg as hdbg diff --git a/helpers/hdocker.py b/helpers/hdocker.py index 7e5b456f5..d5f0fcbb3 100644 --- a/helpers/hdocker.py +++ b/helpers/hdocker.py @@ -868,8 +868,8 @@ def convert_pandoc_arguments_to_cmd( hdbg.dassert_is_subset( params.keys(), ["input", "output", "in_dir_params", "cmd_opts"] ) - cmd.append(f'{params["input"]}') - cmd.append(f'--output {params["output"]}') + cmd.append(f"{params['input']}") + cmd.append(f"--output {params['output']}") for key, value in params["in_dir_params"].items(): if value: cmd.append(f"--{key} {value}") @@ -1232,7 +1232,7 @@ def convert_latex_arguments_to_cmd( cmd.append(" ".join(params["cmd_opts"])) # The input needs to be last to work around the bug in pdflatex where the # options before the input file are not always parsed correctly. - cmd.append(f'{params["input"]}') + cmd.append(f"{params['input']}") # cmd = " ".join(cmd) _LOG.debug(hprint.to_str("cmd")) diff --git a/helpers/henv.py b/helpers/henv.py index 7317ca95b..6beb6e0d7 100644 --- a/helpers/henv.py +++ b/helpers/henv.py @@ -303,7 +303,7 @@ def _git_log(num_commits: int = 5, my_commits: bool = False) -> str: cmd = [] cmd.append("git log --date=local --oneline --graph --date-order --decorate") cmd.append( - "--pretty=format:" "'%h %<(8)%aN% %<(65)%s (%>(14)%ar) %ad %<(10)%d'" + "--pretty=format:'%h %<(8)%aN% %<(65)%s (%>(14)%ar) %ad %<(10)%d'" ) cmd.append(f"-{num_commits}") if my_commits: diff --git a/helpers/hgit.py b/helpers/hgit.py index 82de76a7a..89e85d7d6 100644 --- a/helpers/hgit.py +++ b/helpers/hgit.py @@ -802,7 +802,11 @@ def find_file_in_git_tree( _, file_name_out = hsystem.system_to_one_line(cmd) _LOG.debug(hprint.to_str("file_name_out")) hdbg.dassert_ne( - file_name_out, "", "Can't find file '%s' in dir '%s'", file_name, root_dir + file_name_out, + "", + "Can't find file '%s' in dir '%s'", + file_name, + root_dir, ) file_name_out: str = os.path.abspath(file_name_out) hdbg.dassert_path_exists(file_name_out) diff --git a/helpers/hgoogle_drive_api.py b/helpers/hgoogle_drive_api.py index 235f337bb..43830cb2c 100644 --- a/helpers/hgoogle_drive_api.py +++ b/helpers/hgoogle_drive_api.py @@ -57,7 +57,9 @@ def get_credentials( # /home/.config/gspread_pandas/google_secret.json # Instructions: https://gspread-pandas.readthedocs.io/en/latest/getting_started.html#client-credentials" hdbg.dassert_file_exists( - service_key_path, "Failed to read service key file: %s", service_key_path + service_key_path, + "Failed to read service key file: %s", + service_key_path, ) # Scopes required for making API calls. scopes = [ diff --git a/helpers/hintrospection.py b/helpers/hintrospection.py index 21867ed98..44fe32811 100644 --- a/helpers/hintrospection.py +++ b/helpers/hintrospection.py @@ -228,7 +228,9 @@ def get_size_in_bytes(obj: object, seen: Optional[set] = None) -> int: for cls in obj.__class__.__mro__: if "__dict__" in cls.__dict__: d = cls.__dict__["__dict__"] - if inspect.isgetsetdescriptor(d) or inspect.ismemberdescriptor(d): + if inspect.isgetsetdescriptor(d) or inspect.ismemberdescriptor( + d + ): size += get_size_in_bytes(obj.__dict__, seen) break if isinstance(obj, dict): diff --git a/helpers/hio.py b/helpers/hio.py index 354cb2c6c..75ffdd1af 100644 --- a/helpers/hio.py +++ b/helpers/hio.py @@ -268,7 +268,9 @@ def create_dir( dir_timestamp = os.path.getmtime(dir_name) dir_datetime = datetime.datetime.fromtimestamp(dir_timestamp) # Build new dir name with timestamp. - dir_name_new = dir_name + "." + dir_datetime.strftime("%Y%m%d_%H%M%S") + dir_name_new = ( + dir_name + "." + dir_datetime.strftime("%Y%m%d_%H%M%S") + ) # Rename dir. if not os.path.exists(dir_name_new): _LOG.warning("Renaming dir '%s' -> '%s'", dir_name, dir_name_new) diff --git a/helpers/hjoblib.py b/helpers/hjoblib.py index d778a15ad..b3bf79a7c 100644 --- a/helpers/hjoblib.py +++ b/helpers/hjoblib.py @@ -567,7 +567,9 @@ def _parallel_execute_decorator( if abort_on_error: _LOG.error("Aborting since abort_on_error=%s", abort_on_error) raise exception # noqa: F821 - _LOG.error("Continuing execution since abort_on_error=%s", abort_on_error) + _LOG.error( + "Continuing execution since abort_on_error=%s", abort_on_error + ) res = str(exception) else: # The execution was successful. diff --git a/helpers/hjupyter.py b/helpers/hjupyter.py index 079796735..5f7d74df0 100644 --- a/helpers/hjupyter.py +++ b/helpers/hjupyter.py @@ -45,7 +45,9 @@ def run_notebook( cmd.append(f"jupytext --update --to notebook {file_name};") notebook_name = f"{root}.ipynb" else: - raise ValueError(f"Unsupported file format for `file_name`='{file_name}'") + raise ValueError( + f"Unsupported file format for `file_name`='{file_name}'" + ) # Execute notebook. cmd.append(f"cd {scratch_dir} &&") cmd.append(f"jupyter nbconvert {notebook_name}") @@ -60,7 +62,11 @@ def run_notebook( def build_run_notebook_cmd( - config_builder: str, dst_dir: str, notebook_path: str, *, extra_opts: str = "" + config_builder: str, + dst_dir: str, + notebook_path: str, + *, + extra_opts: str = "", ) -> str: """ Construct a command string to run dev_scripts/notebooks/run_notebook.py diff --git a/helpers/hlatex.py b/helpers/hlatex.py index 98da2cb75..33fe3b270 100644 --- a/helpers/hlatex.py +++ b/helpers/hlatex.py @@ -21,9 +21,7 @@ def convert_pandoc_md_to_latex(txt: str) -> str: # Run Pandoc. out_file_name = "./tmp.run_pandoc_out.tex" cmd = ( - f"pandoc {in_file_name}" - f" -o {out_file_name}" - " --read=markdown --write=latex" + f"pandoc {in_file_name} -o {out_file_name} --read=markdown --write=latex" ) container_type = "pandoc_only" hdocker.run_dockerized_pandoc(cmd, container_type) diff --git a/helpers/hlist.py b/helpers/hlist.py index 31ad2396f..137d8e4ee 100644 --- a/helpers/hlist.py +++ b/helpers/hlist.py @@ -8,6 +8,7 @@ import helpers.hdbg as hdbg + # TODO(gp): -> return_single_element, return_single_element_or_assert? def assert_single_element_and_return(list_: List[Any]) -> Any: """ diff --git a/helpers/hlogging.py b/helpers/hlogging.py index c83bcf68d..69a58343a 100644 --- a/helpers/hlogging.py +++ b/helpers/hlogging.py @@ -184,8 +184,7 @@ def shutup_chatty_modules( "invoke", "matplotlib", "nose", - "openai" - "s3fs", + "openais3fs", "s3transfer", "urllib3", # "ib_insync", @@ -227,7 +226,7 @@ def __init__(self, *args: Any, **kwargs: Any): self._tzinfo = pytz.timezone("America/New_York") except ModuleNotFoundError: - #print(f"Can't import pytz: using UTC\n{str(e)}") + # print(f"Can't import pytz: using UTC\n{str(e)}") self._tzinfo = None def converter(self, timestamp: float) -> datetime.datetime: @@ -558,7 +557,7 @@ def __init__( self._tzinfo = pytz.timezone("America/New_York") except ModuleNotFoundError: - #print(f"Can't import pytz: using UTC\n{str(e)}") + # print(f"Can't import pytz: using UTC\n{str(e)}") self._tzinfo = None # self._report_memory_usage = report_memory_usage diff --git a/helpers/hmarkdown.py b/helpers/hmarkdown.py index 53429795f..765a755c3 100644 --- a/helpers/hmarkdown.py +++ b/helpers/hmarkdown.py @@ -486,7 +486,9 @@ def sanity_check_header_list(header_list: HeaderList) -> None: hdbg.dassert_isinstance(header_list[i], HeaderInfo) if header_list[i].level - header_list[i - 1].level > 1: msg = [] - msg.append("Consecutive headers increase by more than one level:") + msg.append( + "Consecutive headers increase by more than one level:" + ) msg.append(f" {header_list[i - 1]}") msg.append(f" {header_list[i]}") msg = "\n".join(msg) @@ -964,7 +966,9 @@ def format_headers(in_file_name: str, out_file_name: str, max_lev: int) -> None: hparser.write_file(txt_tmp, out_file_name) -def modify_header_level(in_file_name: str, out_file_name: str, mode: str) -> None: +def modify_header_level( + in_file_name: str, out_file_name: str, mode: str +) -> None: """ Increase or decrease the level of headings by one for text in stdin. @@ -1133,7 +1137,10 @@ def selected_navigation_to_str( ) _LOG.debug(hprint.to_str("ancestry")) txt = header_tree_to_str( - tree, ancestry, open_modifier=open_modifier, close_modifier=close_modifier + tree, + ancestry, + open_modifier=open_modifier, + close_modifier=close_modifier, ) return txt diff --git a/helpers/hnumba.py b/helpers/hnumba.py index af6ff696c..47fc37975 100644 --- a/helpers/hnumba.py +++ b/helpers/hnumba.py @@ -27,7 +27,6 @@ def jit(f: Callable[..., RT]) -> Callable[..., RT]: - if USE_NUMBA and not numba_available: _LOG.warning("numba is not installed") use_numba = USE_NUMBA and numba_available diff --git a/helpers/hnumpy.py b/helpers/hnumpy.py index 911f40f8e..4cd0e8c4d 100644 --- a/helpers/hnumpy.py +++ b/helpers/hnumpy.py @@ -11,6 +11,7 @@ import helpers.hdbg as hdbg + # From https://stackoverflow.com/questions/49555991 @contextlib.contextmanager def random_seed_context(seed: int) -> Iterator: @@ -29,16 +30,16 @@ def random_seed_context(seed: int) -> Iterator: def floor_with_precision(value: float, amount_precision: int) -> float: """ Floor a value using desired precision. - + The invariant for this function is that negative number are floored based on their absolute value: e.g floor_with_precision(-4.6, 0) == -4. This is useful for calculating share size where there are decimal precision limitations. The desired behavior is to rather round down than overfill. - + Other examples: floor_with_precision(0.125, 2) == 0.12 floor_with_precision(0.4, 0) == 0.0 - + :param value: value to floor with desire :param amount_precision: number of decimal points to floor to :return: value floored using desired precision. @@ -51,7 +52,6 @@ def floor_with_precision(value: float, amount_precision: int) -> float: sign = -1 if value < 0 else 1 value_abs = abs(value) value_floored = np.true_divide( - np.floor(value_abs * 10**amount_precision), - 10**amount_precision + np.floor(value_abs * 10**amount_precision), 10**amount_precision ) return value_floored * sign diff --git a/helpers/hopen.py b/helpers/hopen.py index c14dbaeb6..7b5e6f5d9 100644 --- a/helpers/hopen.py +++ b/helpers/hopen.py @@ -35,7 +35,9 @@ def _cmd_open_html(file_name: str, os_name: str) -> Optional[str]: hdbg.dassert_in(os_name, os_cmds) exec_name = os_cmds[os_name] if not hsystem.check_exec(exec_name): - _LOG.warning("Can't execute the command '%s' on this platform", exec_name) + _LOG.warning( + "Can't execute the command '%s' on this platform", exec_name + ) return None # Build the command. full_cmd = f"{exec_name} {file_name}" diff --git a/helpers/hopenai.py b/helpers/hopenai.py index 93eea6580..239b14d96 100644 --- a/helpers/hopenai.py +++ b/helpers/hopenai.py @@ -14,8 +14,9 @@ from typing import Any, Dict, List, Tuple import openai -#import openai.types.beta.assistant as OAssistant -#import openai.types.beta.threads.message as OMessage + +# import openai.types.beta.assistant as OAssistant +# import openai.types.beta.threads.message as OMessage import pandas as pd import requests import tqdm @@ -174,7 +175,8 @@ def _retrieve_openrouter_model_info() -> pd.DataFrame: def _save_models_info_to_csv( - model_info_df: pd.DataFrame, file_name: str, + model_info_df: pd.DataFrame, + file_name: str, ) -> pd.DataFrame: """ Save models info to a CSV file. @@ -344,7 +346,7 @@ def get_current_cost() -> float: def _calculate_cost( completion: openai.types.chat.chat_completion.ChatCompletion, model: str, - models_info_file: str + models_info_file: str, ) -> float: """ Calculate the cost of an OpenAI API call. @@ -388,7 +390,9 @@ def _calculate_cost( prompt_price = row["prompt_pricing"] completion_price = row["completion_pricing"] # Compute cost. - cost = prompt_tokens * prompt_price + completion_tokens * completion_price + cost = ( + prompt_tokens * prompt_price + completion_tokens * completion_price + ) else: raise ValueError(f"Unknown provider: {_PROVIDER_NAME}") _LOG.debug(hprint.to_str("prompt_tokens completion_tokens cost")) @@ -786,7 +790,7 @@ def apply_prompt_to_dataframe( response = get_completion(user, system=prompt, model=model) except Exception as e: _LOG.error( - f"Error processing column {input} in chunk" f" {start}-{end}: {e}" + f"Error processing column {input} in chunk {start}-{end}: {e}" ) raise e processed_response = response.split("\n") @@ -892,9 +896,9 @@ def save_response_to_cache( """ entry = {"request": request, "response": response} self.cache["entries"][hash_key] = entry - self.cache["metadata"][ - "last_updated" - ] = datetime.datetime.now().isoformat() + self.cache["metadata"]["last_updated"] = ( + datetime.datetime.now().isoformat() + ) self._write_cache_to_disk() def load_response_from_cache(self, hash_key: str) -> Any: @@ -933,7 +937,7 @@ def _clear_cache(self) -> None: Clear the cache from the file. """ self.cache["entries"] = {} - self.cache["metadata"][ - "last_updated" - ] = datetime.datetime.now().isoformat() + self.cache["metadata"]["last_updated"] = ( + datetime.datetime.now().isoformat() + ) self._write_cache_to_disk() diff --git a/helpers/hpandas.py b/helpers/hpandas.py index 8a05d0d90..846288761 100644 --- a/helpers/hpandas.py +++ b/helpers/hpandas.py @@ -1146,7 +1146,10 @@ def _display(log_level: int, df: pd.DataFrame) -> None: """ from IPython.display import display - if hsystem.is_running_in_ipynb() and log_level >= hdbg.get_logger_verbosity(): + if ( + hsystem.is_running_in_ipynb() + and log_level >= hdbg.get_logger_verbosity() + ): display(df) @@ -1844,7 +1847,9 @@ def remove_outliers( hdbg.dassert_is_subset(columns, df.columns) for column in all_columns: if column in columns: - df[column] = df[column].quantile([lower_quantile, upper_quantile]) + df[column] = df[column].quantile( + [lower_quantile, upper_quantile] + ) elif axis == 1: all_rows = df.rows rows = _resolve_column_names(column_set, all_rows) @@ -1997,8 +2002,12 @@ def compare_dfs( raise ValueError(f"Invalid column_mode='{column_mode}'") # Round small numbers to 0 to exclude them from the diff computation. close_to_zero_threshold_mask = lambda x: abs(x) < close_to_zero_threshold - df1[close_to_zero_threshold_mask] = df1[close_to_zero_threshold_mask].round(0) - df2[close_to_zero_threshold_mask] = df2[close_to_zero_threshold_mask].round(0) + df1[close_to_zero_threshold_mask] = df1[close_to_zero_threshold_mask].round( + 0 + ) + df2[close_to_zero_threshold_mask] = df2[close_to_zero_threshold_mask].round( + 0 + ) # Compute the difference df. if diff_mode == "diff": # Test and convert the assertion into a boolean. @@ -2044,7 +2053,9 @@ def compare_dfs( # Check if `df_diff` values are less than `assert_diff_threshold`. if assert_diff_threshold is not None: nan_mask = df_diff.isna() - within_threshold = (df_diff.abs() <= assert_diff_threshold) | nan_mask + within_threshold = ( + df_diff.abs() <= assert_diff_threshold + ) | nan_mask expected = pd.DataFrame( True, index=within_threshold.index, diff --git a/helpers/hparquet.py b/helpers/hparquet.py index f584b86a9..6957f7d34 100644 --- a/helpers/hparquet.py +++ b/helpers/hparquet.py @@ -119,7 +119,9 @@ def _get_core_dataframes(self) -> List[pd.DataFrame]: index=self._dataframe_index, ) _LOG.debug( - hpandas.df_to_str(asset_df, print_shape_info=True, tag="asset_df") + hpandas.df_to_str( + asset_df, print_shape_info=True, tag="asset_df" + ) ) df.append(asset_df) return df diff --git a/helpers/hparser.py b/helpers/hparser.py index f6bcb7bef..10fc5bfc7 100644 --- a/helpers/hparser.py +++ b/helpers/hparser.py @@ -212,7 +212,9 @@ def actions_to_string( def select_actions( - args: argparse.Namespace, valid_actions: List[str], default_actions: List[str] + args: argparse.Namespace, + valid_actions: List[str], + default_actions: List[str], ) -> List[str]: hdbg.dassert( not (args.action and args.all), @@ -675,9 +677,7 @@ def add_dockerized_script_arg( def add_llm_prompt_arg( - parser: argparse.ArgumentParser, - *, - default_prompt: str = "" + parser: argparse.ArgumentParser, *, default_prompt: str = "" ) -> argparse.ArgumentParser: """ Add common command line arguments for `*llm_transform.py` scripts. diff --git a/helpers/hpickle.py b/helpers/hpickle.py index 33c65612c..9d274daa2 100644 --- a/helpers/hpickle.py +++ b/helpers/hpickle.py @@ -133,7 +133,9 @@ def from_pickle( Unpickle and return object stored in `file_name`. """ hdbg.dassert_isinstance(file_name, str) - with htimer.TimedScope(logging.DEBUG, f"Unpickling from '{file_name}'") as ts: + with htimer.TimedScope( + logging.DEBUG, f"Unpickling from '{file_name}'" + ) as ts: # We assume that the user always specifies a .pkl extension and then we # change the extension based on the backend. if backend in ("pickle", "dill"): diff --git a/helpers/hplayback.py b/helpers/hplayback.py index c6092fc94..48c8c4f08 100644 --- a/helpers/hplayback.py +++ b/helpers/hplayback.py @@ -147,14 +147,14 @@ def __init__( x = self._kwargs.pop("self") self._parent_class = x self._code = [ - f'# Test created for {cur_frame.f_back.f_globals["__name__"]}' # type: ignore + f"# Test created for {cur_frame.f_back.f_globals['__name__']}" # type: ignore f".{x.__class__.__name__}.{self._func_name}." ] else: self._parent_class = None self._code = [ # pylint: disable=line-too-long - f'# Test created for {cur_frame.f_back.f_globals["__name__"]}.{self._func_name}.' # type: ignore + f"# Test created for {cur_frame.f_back.f_globals['__name__']}.{self._func_name}." # type: ignore ] self._append("") # Check if need to write the code directly to file. @@ -247,7 +247,9 @@ def _check_code(self, func_output: Any) -> None: if self.mode == "check_string": if isinstance(func_output, (pd.DataFrame, pd.Series, str)): if not isinstance(func_output, str): - self._append("act = hpandas.df_to_str(act, num_rows=None)", 2) + self._append( + "act = hpandas.df_to_str(act, num_rows=None)", 2 + ) if not isinstance(func_output, (str, bytes)): self._append("act = str(act)", 2) self._append("# Check output.", 2) @@ -320,7 +322,9 @@ def _get_class_name_string(self) -> str: if self._parent_class is not None else "" ) - test_name += "".join([x.capitalize() for x in self._func_name.split("_")]) + test_name += "".join( + [x.capitalize() for x in self._func_name.split("_")] + ) class_string = f"class Test{test_name}(hunitest.TestCase):" return class_string @@ -339,7 +343,9 @@ def _add_function_call(self) -> None: self._append("cls = jsonpickle.decode(cls)", 2) fnc_call = [f"{k}={k}" for k in self._kwargs.keys()] # Call the method as a child of the parent class. - self._append(f"act = cls.{self._func_name}({', '.join(fnc_call)})", 2) + self._append( + f"act = cls.{self._func_name}({', '.join(fnc_call)})", 2 + ) def _add_var_definitions(self) -> None: """ diff --git a/helpers/hprint.py b/helpers/hprint.py index 61f79b549..82689bb0d 100644 --- a/helpers/hprint.py +++ b/helpers/hprint.py @@ -288,7 +288,9 @@ def dedent(txt: str, *, remove_lead_trail_empty_lines_: bool = True) -> str: # Find the minimum number of leading spaces. min_num_spaces = None for curr_line in txt.split("\n"): - _LOG.debug("min_num_spaces=%s: curr_line='%s'", min_num_spaces, curr_line) + _LOG.debug( + "min_num_spaces=%s: curr_line='%s'", min_num_spaces, curr_line + ) # Skip empty lines. if curr_line.lstrip().rstrip() == "": _LOG.debug(" -> Skipping empty line") @@ -973,8 +975,10 @@ def filter_text(regex: str, txt: str) -> str: def dassert_one_trailing_newline(txt: str) -> None: - num_newlines = len(re.search(r'\n*$', txt).group()) - hdbg.dassert_eq(num_newlines, 0, "num_newlines='%s' txt='%s'", num_newlines, txt) + num_newlines = len(re.search(r"\n*$", txt).group()) + hdbg.dassert_eq( + num_newlines, 0, "num_newlines='%s' txt='%s'", num_newlines, txt + ) def to_info(tag: str, txt: Union[str, List[str]]) -> str: diff --git a/helpers/hpytest.py b/helpers/hpytest.py index 23de719d1..30c781a78 100644 --- a/helpers/hpytest.py +++ b/helpers/hpytest.py @@ -16,7 +16,9 @@ _LOG = logging.getLogger(__name__) -def _pytest_show_artifacts(dir_name: str, tag: Optional[str] = None) -> List[str]: +def _pytest_show_artifacts( + dir_name: str, tag: Optional[str] = None +) -> List[str]: hdbg.dassert_ne(dir_name, "") hdbg.dassert_dir_exists(dir_name) cd_cmd = f"cd {dir_name} && " diff --git a/helpers/hretry.py b/helpers/hretry.py index 1243dcb3c..2ee2166f9 100644 --- a/helpers/hretry.py +++ b/helpers/hretry.py @@ -3,6 +3,7 @@ import helpers.hretry as hretry """ + import asyncio import functools import logging @@ -46,7 +47,9 @@ def retry_wrapper(*args, **kwargs): ) attempts_count += 1 time.sleep(retry_delay_in_sec) - _LOG.error("Function %s failed after %d attempts", func, num_attempts) + _LOG.error( + "Function %s failed after %d attempts", func, num_attempts + ) raise last_exception return retry_wrapper @@ -81,7 +84,9 @@ async def retry_wrapper(*args, **kwargs): ) attempts_count += 1 await asyncio.sleep(retry_delay_in_sec) - _LOG.error("Function %s failed after %d attempts", func, num_attempts) + _LOG.error( + "Function %s failed after %d attempts", func, num_attempts + ) raise last_exception return retry_wrapper diff --git a/helpers/hs3.py b/helpers/hs3.py index 31c593af0..6ea1d3210 100644 --- a/helpers/hs3.py +++ b/helpers/hs3.py @@ -666,7 +666,9 @@ def generate_aws_files( config_file_name = os.path.join(home_dir, ".aws", "config") credentials_file_name = os.path.join(home_dir, ".aws", "credentials") # Check if the files already exist. - if os.path.exists(credentials_file_name) and os.path.exists(config_file_name): + if os.path.exists(credentials_file_name) and os.path.exists( + config_file_name + ): _LOG.info( "Both files exist: %s and %s; exiting", credentials_file_name, @@ -911,7 +913,9 @@ def archive_data_on_s3( hdbg.dassert_dir_exists(src_dir) dassert_is_s3_path(s3_path) _LOG.info( - "The size of '%s' is %s", src_dir, hsystem.du(src_dir, human_format=True) + "The size of '%s' is %s", + src_dir, + hsystem.du(src_dir, human_format=True), ) # Add a timestamp if needed. dst_path = hsystem.append_timestamp_tag(src_dir, tag) + ".tgz" diff --git a/helpers/hsecrets.py b/helpers/hsecrets.py index 2c8525466..8b1568f35 100644 --- a/helpers/hsecrets.py +++ b/helpers/hsecrets.py @@ -75,7 +75,8 @@ def lock_secret( # Raise warning of locked resource with current use info. # raise RuntimeError() warnings.warn( - f"Secret key is already in use by {current_usedBy[0]}", RuntimeWarning + f"Secret key is already in use by {current_usedBy[0]}", + RuntimeWarning, ) return secret_value diff --git a/helpers/hsql.py b/helpers/hsql.py index 4468063e4..897394502 100644 --- a/helpers/hsql.py +++ b/helpers/hsql.py @@ -29,6 +29,8 @@ def create_in_operator(values: List[str], column_name: str) -> str: e.g. `"exchange_id IN ('binance', 'ftx')"` """ in_operator = ( - f"{column_name} IN (" + ",".join([f"'{value}'" for value in values]) + ")" + f"{column_name} IN (" + + ",".join([f"'{value}'" for value in values]) + + ")" ) return in_operator diff --git a/helpers/hsql_implementation.py b/helpers/hsql_implementation.py index 468aa81db..dbb73aaba 100644 --- a/helpers/hsql_implementation.py +++ b/helpers/hsql_implementation.py @@ -364,7 +364,9 @@ def remove_database(connection: DbConnection, dbname: str) -> None: # Drop database. # From https://stackoverflow.com/questions/36502401 connection.cursor().execute( - psql.SQL("DROP DATABASE {} WITH (FORCE);").format(psql.Identifier(dbname)) + psql.SQL("DROP DATABASE {} WITH (FORCE);").format( + psql.Identifier(dbname) + ) ) @@ -711,7 +713,9 @@ def create_insert_on_conflict_do_nothing_query( # TODO(gp): -> connection, table_name, obj def execute_insert_query( - connection: DbConnection, obj: Union[pd.DataFrame, pd.Series], table_name: str + connection: DbConnection, + obj: Union[pd.DataFrame, pd.Series], + table_name: str, ) -> None: """ Insert a DB as multiple rows into the database. @@ -727,7 +731,7 @@ def execute_insert_query( hdbg.dassert_isinstance(df, pd.DataFrame) hdbg.dassert_in(table_name, get_table_names(connection)) _LOG.debug("df=\n%s", hpandas.df_to_str(df, use_tabulate=False)) - # Ensure the DataFrame has compatible types with + # Ensure the DataFrame has compatible types with # downstream consumers (e.g., database). df = df.applymap(lambda x: float(x) if isinstance(x, np.float64) else x) # Transform dataframe into list of tuples. diff --git a/helpers/hsystem.py b/helpers/hsystem.py index 80ed95326..e99e678d8 100644 --- a/helpers/hsystem.py +++ b/helpers/hsystem.py @@ -202,7 +202,11 @@ def _system( # with hloggin.set_level(_LOG, logging.DEBUG): # _LOG.debug("> %s", cmd) with subprocess.Popen( - cmd, shell=True, executable="/bin/bash", stdout=stdout, stderr=stderr + cmd, + shell=True, + executable="/bin/bash", + stdout=stdout, + stderr=stderr, ) as p: output = "" if blocking: @@ -567,7 +571,9 @@ def get_process_pids( try: pid = int(fields[0]) except ValueError as e: - _LOG.error("Can't parse fields '%s' from line '%s'", fields, line) + _LOG.error( + "Can't parse fields '%s' from line '%s'", fields, line + ) raise e _LOG.debug("pid=%s", pid) pids.append(pid) @@ -902,7 +908,9 @@ def find_file_with_dir( file_name = file_name[len(prefix) :] # Count how many dirs levels there are. dir_depth = len(os.path.normpath(file_name).split("/")) - 1 - _LOG.debug("inferred dir_depth=%s for file_name=%s", dir_depth, file_name) + _LOG.debug( + "inferred dir_depth=%s for file_name=%s", dir_depth, file_name + ) # Check the matching files. matching_files = [] for candidate_file_name in sorted(candidate_files): @@ -957,7 +965,9 @@ def has_timestamp(file_name: str) -> bool: # E.g., %Y%m%d-%H_%M_%S # The separator is _, -, or nothing. sep = "[-_]?" - regex = sep.join([r"\d{4}", r"\d{2}", r"\d{2}", r"\d{2}", r"\d{2}", r"\d{2}"]) + regex = sep.join( + [r"\d{4}", r"\d{2}", r"\d{2}", r"\d{2}", r"\d{2}", r"\d{2}"] + ) _LOG.debug("regex=%s", regex) occurrences = re.findall(regex, file_name) hdbg.dassert_lte( diff --git a/helpers/htest_logger.py b/helpers/htest_logger.py index ee336e040..8ef0e3a4f 100644 --- a/helpers/htest_logger.py +++ b/helpers/htest_logger.py @@ -11,7 +11,6 @@ import dev_scripts_helpers.script_template as dscscske """ - import argparse import logging @@ -26,7 +25,8 @@ def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("positional", nargs="*", help="...") parser.add_argument("--dst_dir", action="store", help="Destination dir") diff --git a/helpers/htimer.py b/helpers/htimer.py index a4b936844..d88ae4e69 100644 --- a/helpers/htimer.py +++ b/helpers/htimer.py @@ -97,7 +97,9 @@ def resume(self) -> None: self._start = time.time() def is_started(self) -> bool: - return self._start is not None and self._start >= 0 and self._stop is None + return ( + self._start is not None and self._start >= 0 and self._stop is None + ) def is_stopped(self) -> bool: return self._start is None and self._stop is None diff --git a/helpers/htqdm.py b/helpers/htqdm.py index a524b9b63..82c2a5a1e 100644 --- a/helpers/htqdm.py +++ b/helpers/htqdm.py @@ -3,6 +3,7 @@ import helpers.htqdm as htqdm """ + import io import logging from typing import Any, Optional diff --git a/helpers/htraceback.py b/helpers/htraceback.py index 0bb676fa4..caa61d4ff 100644 --- a/helpers/htraceback.py +++ b/helpers/htraceback.py @@ -179,7 +179,7 @@ def parse_traceback( end_idx += 1 line = lines[end_idx] _LOG.debug( - "Extend traceback: to_break=%s, end_idx=%s, " "line='%s'", + "Extend traceback: to_break=%s, end_idx=%s, line='%s'", to_break, end_idx, line, diff --git a/helpers/htranslate.py b/helpers/htranslate.py index d446566bc..db1a4357c 100644 --- a/helpers/htranslate.py +++ b/helpers/htranslate.py @@ -11,6 +11,7 @@ import helpers.htranslate as htransl """ + import argparse import configparser import logging diff --git a/helpers/hunit_test.py b/helpers/hunit_test.py index ccc1cf122..ab97bcf18 100644 --- a/helpers/hunit_test.py +++ b/helpers/hunit_test.py @@ -459,7 +459,10 @@ def purify_app_references(txt: str) -> str: txt = re.sub(r"app\.helpers", "helpers", txt, flags=re.MULTILINE) txt = re.sub(r"app\.amp\.helpers", "amp.helpers", txt, flags=re.MULTILINE) txt = re.sub( - r"app\.amp\.helpers_root\.helpers", "amp.helpers", txt, flags=re.MULTILINE + r"app\.amp\.helpers_root\.helpers", + "amp.helpers", + txt, + flags=re.MULTILINE, ) _LOG.debug("After %s: txt='\n%s'", hintros.get_function_name(), txt) return txt @@ -520,7 +523,10 @@ def purify_today_date(txt: str) -> str: today_date_as_str = today_date.strftime("%Y%m%d") # Replace predict.3.compress_tails.df_out.20220627_094500.YYYYMMDD_171106.csv.gz. txt = re.sub( - today_date_as_str + r"_\d{6}", "YYYYMMDD_HHMMSS", txt, flags=re.MULTILINE + today_date_as_str + r"_\d{6}", + "YYYYMMDD_HHMMSS", + txt, + flags=re.MULTILINE, ) txt = re.sub(today_date_as_str, "YYYYMMDD", txt, flags=re.MULTILINE) return txt @@ -1579,7 +1585,9 @@ def check_string( # Create golden file and add it to the repo. _LOG.warning("Creating the golden outcome") outcome_updated = True - self._check_string_update_outcome(file_name, actual, use_gzip) + self._check_string_update_outcome( + file_name, actual, use_gzip + ) is_equal = None else: hdbg.dfatal( @@ -1721,7 +1729,8 @@ def check_df_output( if expected_column_names: # Verify that the column names are correct. self.assert_equal( - str(sorted(actual_df.columns)), str(sorted(expected_column_names)) + str(sorted(actual_df.columns)), + str(sorted(expected_column_names)), ) if expected_column_unique_values: hdbg.dassert_is_subset( diff --git a/helpers/hunit_test_utils.py b/helpers/hunit_test_utils.py index 0d0ddcbb2..808472c83 100644 --- a/helpers/hunit_test_utils.py +++ b/helpers/hunit_test_utils.py @@ -323,7 +323,9 @@ def _rename_method( new_content = "\n".join(lines) return new_content, num_replaced - def _process_outcomes_dir(self, outcome_dir: str, outcomes_path: str) -> bool: + def _process_outcomes_dir( + self, outcome_dir: str, outcomes_path: str + ) -> bool: """ Process the directory containing target test outcomes. @@ -417,7 +419,9 @@ def run_test_to_config_str(self, obj: Any, expected_str: str) -> None: method_name = "to_config_str" self._test_method(obj, method_name, expected_str) - def _test_method(self, obj: Any, method_name: str, expected_str: str) -> None: + def _test_method( + self, obj: Any, method_name: str, expected_str: str + ) -> None: """ Common method for testing `__repr__` and `__str__`. """ diff --git a/helpers/lib_tasks.py b/helpers/lib_tasks.py index 12c9903fd..d31d0bb3a 100644 --- a/helpers/lib_tasks.py +++ b/helpers/lib_tasks.py @@ -11,7 +11,7 @@ # Import this way to avoid complexity in propagating the refactoring in all # the repos downstream. -from helpers.lib_tasks_aws import * # isort:skip # noqa: F401,F403 # pylint: disable=unused-import,unused-wildcard-import,wildcard-import +from helpers.lib_tasks_aws import * # isort:skip # noqa: F401,F403 # pylint: disable=unused-import,unused-wildcard-import,wildcard-import from helpers.lib_tasks_bash import * # isort:skip # noqa: F401,F403 # pylint: disable=unused-import,unused-wildcard-import,wildcard-import from helpers.lib_tasks_docker import * # isort:skip # noqa: F401,F403 # pylint: disable=unused-import,unused-wildcard-import,wildcard-import from helpers.lib_tasks_docker_release import * # isort:skip # noqa: F401,F403 # pylint: disable=unused-import,unused-wildcard-import,wildcard-import diff --git a/helpers/lib_tasks_aws.py b/helpers/lib_tasks_aws.py index 998f96205..56785c3e9 100644 --- a/helpers/lib_tasks_aws.py +++ b/helpers/lib_tasks_aws.py @@ -101,7 +101,8 @@ def release_dags_to_airflow( # If same file already exists, then overwrite. if os.path.exists(dest_file): _LOG.warning( - "DAG already exists in destination, Overwriting ... %s", dest_file + "DAG already exists in destination, Overwriting ... %s", + dest_file, ) # Steps to overwrite: # 1. Change user to root. @@ -118,7 +119,10 @@ def release_dags_to_airflow( _LOG.info( "DAG doesn't exist in destination, Copying ... %s", dest_file ) - cmds = [f"cp {file_path} {dest_file}", f"sudo chmod a-w {dest_file}"] + cmds = [ + f"cp {file_path} {dest_file}", + f"sudo chmod a-w {dest_file}", + ] cmd = "&&".join(cmds) # TODO(sonaal): Instead of running scripts, run individual commands. # Append script for each file to a temporary file @@ -189,9 +193,9 @@ def _set_task_definition_config( # We use single container inside our task definition and # the convention is to set the same name as the task # definition itself. - task_definition_config["containerDefinitions"][0][ - "name" - ] = task_definition_name + task_definition_config["containerDefinitions"][0]["name"] = ( + task_definition_name + ) # Set placeholder image URL. image_name = hrecouti.get_repo_config().get_docker_base_image_name() task_definition_config["containerDefinitions"][0]["image"] = ( @@ -213,9 +217,9 @@ def _set_task_definition_config( # Configure access to EFS. efs_config = _get_efs_mount_config_template() task_definition_config["volumes"] = efs_config[region]["volumes"] - task_definition_config["containerDefinitions"][0]["mountPoints"] = efs_config[ - region - ]["mountPoints"] + task_definition_config["containerDefinitions"][0]["mountPoints"] = ( + efs_config[region]["mountPoints"] + ) return task_definition_config @@ -252,7 +256,9 @@ def _register_task_definition(task_definition_name: str, region: str) -> None: placementConstraints=task_definition_config.get( "placementConstraints", [] ), - requiresCompatibilities=task_definition_config["requiresCompatibilities"], + requiresCompatibilities=task_definition_config[ + "requiresCompatibilities" + ], cpu=task_definition_config["cpu"], memory=task_definition_config["memory"], ) diff --git a/helpers/lib_tasks_docker.py b/helpers/lib_tasks_docker.py index c47e1435e..bfdaf55d0 100644 --- a/helpers/lib_tasks_docker.py +++ b/helpers/lib_tasks_docker.py @@ -552,7 +552,9 @@ def _generate_docker_compose_file( ) # A super repo is a repo that contains helpers as a submodule and # is not a helper itself. - use_helpers_as_nested_module = 0 if hgit.is_in_helpers_as_supermodule() else 1 + use_helpers_as_nested_module = ( + 0 if hgit.is_in_helpers_as_supermodule() else 1 + ) # We could do the same also with IMAGE for symmetry. # Keep the env vars in sync with what we print in `henv.get_env_vars()`. # Configure `base_app` service. @@ -1329,7 +1331,9 @@ def _docker_cmd( hs3.generate_aws_files() docker_pull(ctx, skip_pull=skip_pull) _LOG.debug("cmd=%s", docker_cmd_) - rc: Optional[int] = hlitauti.run(ctx, docker_cmd_, pty=True, **ctx_run_kwargs) + rc: Optional[int] = hlitauti.run( + ctx, docker_cmd_, pty=True, **ctx_run_kwargs + ) return rc diff --git a/helpers/lib_tasks_docker_release.py b/helpers/lib_tasks_docker_release.py index a0af454fb..d28f91d21 100644 --- a/helpers/lib_tasks_docker_release.py +++ b/helpers/lib_tasks_docker_release.py @@ -1150,7 +1150,9 @@ def docker_release_all(ctx, version, container_dir_name="."): # type: ignore """ hlitauti.report_task() docker_release_dev_image(ctx, version, container_dir_name=container_dir_name) - docker_release_prod_image(ctx, version, container_dir_name=container_dir_name) + docker_release_prod_image( + ctx, version, container_dir_name=container_dir_name + ) _LOG.info("==> SUCCESS <==") @@ -1225,7 +1227,14 @@ def _check_workspace_dir_sizes() -> None: )[1].split("\n") # Filter out directories ignored by `dockerignore.prod` + "amp/" # as submodule. - ignored_dirs = ["amp", "ck.infra", "amp/ck.infra", "docs", ".git", "amp/.git"] + ignored_dirs = [ + "amp", + "ck.infra", + "amp/ck.infra", + "docs", + ".git", + "amp/.git", + ] offending_items = [ it.replace("\t", " ") for it in directory_size_list @@ -1411,14 +1420,18 @@ def docker_update_prod_task_definition( # Compose new prod image url. new_prod_image_url = hlitadoc.get_image(base_image, stage, prod_version) version = None - new_prod_image_url_no_version = hlitadoc.get_image(base_image, stage, version) + new_prod_image_url_no_version = hlitadoc.get_image( + base_image, stage, version + ) # Check if preprod tag exist in preprod task definition as precaution. preprod_task_definition_name = f"{task_definition}-preprod" preprod_image_url = haws.get_task_definition_image_url( preprod_task_definition_name ) preprod_tag_from_image = preprod_image_url.split(":")[-1] - msg = f"Preprod tag is different in the image url `{preprod_tag_from_image}`!" + msg = ( + f"Preprod tag is different in the image url `{preprod_tag_from_image}`!" + ) hdbg.dassert_eq(preprod_tag_from_image, preprod_tag, msg=msg) # Pull preprod image for re-tag. hlitadoc.docker_login(ctx) @@ -1480,7 +1493,8 @@ def docker_update_prod_task_definition( if len(versions) > 1: rollback_version = versions[1] _LOG.info( - "Active version is now `%s`!", rollback_version.version_id + "Active version is now `%s`!", + rollback_version.version_id, ) elif len(versions) == 1: _LOG.info( diff --git a/helpers/lib_tasks_find.py b/helpers/lib_tasks_find.py index 77531e5e8..58e95bb01 100644 --- a/helpers/lib_tasks_find.py +++ b/helpers/lib_tasks_find.py @@ -121,7 +121,9 @@ def _find_test_class( # TODO(gp): Have a single `find` command with multiple options to search for different # things, e.g., class names, test names, pytest_mark, ... @task -def find_test_class(ctx, class_name, dir_name=".", pbcopy=True, exact_match=False): # type: ignore +def find_test_class( + ctx, class_name, dir_name=".", pbcopy=True, exact_match=False +): # type: ignore """ Report test files containing `class_name` in a format compatible with pytest. @@ -318,7 +320,9 @@ def find(ctx, regex, mode="all", how="remove_dups", subdir="."): # type: ignore # TODO(gp): decorator_name -> pytest_mark -def _find_test_decorator(decorator_name: str, file_names: List[str]) -> List[str]: +def _find_test_decorator( + decorator_name: str, file_names: List[str] +) -> List[str]: """ Find test files containing tests with a certain decorator `@pytest.mark.XYZ`. diff --git a/helpers/lib_tasks_gh.py b/helpers/lib_tasks_gh.py index 35ecea0ba..8532f7d09 100644 --- a/helpers/lib_tasks_gh.py +++ b/helpers/lib_tasks_gh.py @@ -231,7 +231,9 @@ def gh_workflow_list( # type: ignore # to the `PATH` (when inside the container) so we can just use # them without specifying the full path. helpers_root_dir = hgit.find_helpers_root() - file_path = f"{helpers_root_dir}/dev_scripts_helpers/system_tools" + file_path = ( + f"{helpers_root_dir}/dev_scripts_helpers/system_tools" + ) cmd = f"{file_path}/remove_escape_chars.py -i {log_file_name}" hsystem.system(cmd) print(f"# Log is in '{log_file_name}'") @@ -711,7 +713,9 @@ def gh_get_overall_build_status_for_repo( return overall_status -def gh_get_workflow_type_names(repo_name: str, *, sort: bool = True) -> List[str]: +def gh_get_workflow_type_names( + repo_name: str, *, sort: bool = True +) -> List[str]: """ Get a list of workflow names for a given repo. @@ -730,7 +734,8 @@ def gh_get_workflow_type_names(repo_name: str, *, sort: bool = True) -> List[str workflow_names = sorted(workflow_names) # Check for duplicate workflow names. hdbg.dassert_no_duplicates( - workflow_names, "Found duplicate workflow names in repo '%s'" % repo_name + workflow_names, + "Found duplicate workflow names in repo '%s'" % repo_name, ) return workflow_names diff --git a/helpers/lib_tasks_git.py b/helpers/lib_tasks_git.py index 25de1d204..a3691b250 100644 --- a/helpers/lib_tasks_git.py +++ b/helpers/lib_tasks_git.py @@ -57,7 +57,9 @@ def git_fetch_master(ctx): # type: ignore @task -def git_merge_master(ctx, abort_if_not_ff=False, abort_if_not_clean=True, skip_fetch=False): # type: ignore +def git_merge_master( + ctx, abort_if_not_ff=False, abort_if_not_clean=True, skip_fetch=False +): # type: ignore """ Merge `origin/master` into the current branch. @@ -660,7 +662,9 @@ def _git_diff_with_branch( cmd.append(f"--diff-filter={diff_type}") cmd.append(f"--name-only HEAD {hash_}") cmd = " ".join(cmd) - files = hsystem.system_to_files(cmd, dir_name, remove_files_non_present=False) + files = hsystem.system_to_files( + cmd, dir_name, remove_files_non_present=False + ) files = sorted(files) _LOG.debug("%s", "\n".join(files)) # Filter by `file_name`, if needed. diff --git a/helpers/lib_tasks_lint.py b/helpers/lib_tasks_lint.py index ecf15c039..8d216bccf 100644 --- a/helpers/lib_tasks_lint.py +++ b/helpers/lib_tasks_lint.py @@ -120,7 +120,15 @@ def lint_check_python_files( # type: ignore The params have the same meaning as in `_get_files_to_process()`. """ - _ = python_compile, python_execute, modified, branch, last_commit, all_, files + _ = ( + python_compile, + python_execute, + modified, + branch, + last_commit, + all_, + files, + ) # Execute the same command line but inside the container. E.g., # /Users/saggese/src/venv/amp.client_venv/bin/invoke lint_docker_check_python_files --branch cmd_line = hdbg.get_command_line() diff --git a/helpers/lib_tasks_perms.py b/helpers/lib_tasks_perms.py index ab0f1f768..215820d4d 100644 --- a/helpers/lib_tasks_perms.py +++ b/helpers/lib_tasks_perms.py @@ -89,7 +89,9 @@ def _find_files_for_user(dir_name: str, user: str, is_equal: bool) -> List[str]: return files -def _find_files_for_group(dir_name: str, group: str, is_equal: bool) -> List[str]: +def _find_files_for_group( + dir_name: str, group: str, is_equal: bool +) -> List[str]: """ Find all the files under `abs_dir` that are owned by a group `group`. """ diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index e3f004e0b..15c7d0ea0 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -150,7 +150,9 @@ def _build_run_command_line( timeout_in_sec = _TEST_TIMEOUTS_IN_SECS[test_list_name] # Detect if we are running on a CK dev server / inside CI # or a laptop outside the CK infra. - is_outside_ck_infra = not hserver.is_dev_csfy() and not hserver.is_inside_ci() + is_outside_ck_infra = ( + not hserver.is_dev_csfy() and not hserver.is_inside_ci() + ) if is_outside_ck_infra: timeout_multiplier = 10 _LOG.warning( @@ -401,7 +403,9 @@ def _get_custom_marker( """ # If we are running outside the CK server / CI, tests requiring CK infra # should be automatically skipped. - is_outside_ck_infra = not hserver.is_dev_csfy() and not hserver.is_inside_ci() + is_outside_ck_infra = ( + not hserver.is_dev_csfy() and not hserver.is_inside_ci() + ) # Skip tests that requires CK infra. if is_outside_ck_infra: _LOG.warning( @@ -1636,7 +1640,9 @@ def _parse_failed_tests( @task -def pytest_failed(ctx, only_file=False, only_class=False, file_name="tmp.pytest_script.txt"): # type: ignore +def pytest_failed( + ctx, only_file=False, only_class=False, file_name="tmp.pytest_script.txt" +): # type: ignore _ = ctx hlitauti.report_task() # Read file. diff --git a/helpers/logging_testing/logging_main.py b/helpers/logging_testing/logging_main.py index 8b28dfd18..631a68e5f 100644 --- a/helpers/logging_testing/logging_main.py +++ b/helpers/logging_testing/logging_main.py @@ -25,7 +25,7 @@ def install_basic_formatter() -> None: def _install_formatter( - formatter: Union[hloggin.CustomFormatter, logging.Formatter] + formatter: Union[hloggin.CustomFormatter, logging.Formatter], ) -> None: root_logger_ = logging.getLogger() ch = logging.StreamHandler(sys.stdout) diff --git a/helpers/notebooks/Master.how_to_use_hgoogle_drive_api.ipynb b/helpers/notebooks/Master.how_to_use_hgoogle_drive_api.ipynb index a874fb208..df337d8b1 100644 --- a/helpers/notebooks/Master.how_to_use_hgoogle_drive_api.ipynb +++ b/helpers/notebooks/Master.how_to_use_hgoogle_drive_api.ipynb @@ -1,424 +1,434 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "CONTENTS:\n", - "- [hgoogle_file_api.py](#hgoogle_file_api.py)\n", - " - [Get Credentials for your drive](#get-credentials-for-your-drive)\n", - " - [Get Tab/Sheet id of a particular google sheet](#get-tab/sheet-id-of-a-particular-google-sheet)\n", - " - [Freeze Rows](#freeze-rows)\n", - " - [Change the height of certin rows](#change-the-height-of-certin-rows)\n", - " - [Read some nice data](#read-some-nice-data)\n", - " - [Write this nice data](#write-this-nice-data)" - ] - }, - { - "cell_type": "markdown", - "id": "982ab891-de0a-47d5-946a-0f4fd3f16307", - "metadata": {}, - "source": [ - "\n", - "# hgoogle_file_api.py" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "6b997caf-4bfc-47bc-b7e1-584f02da328f", - "metadata": {}, - "outputs": [], - "source": [ - "#!sudo /bin/bash -c \"(source /venv/bin/activate; pip install --upgrade google-api-python-client)\"\n", - "# !sudo /bin/bash -c \"(source /venv/bin/activate; pip install --upgrade pip install oauth2client)\"\n", - "#!sudo /bin/bash -c \"(source /venv/bin/activate; pip install --upgrade gspread)\"" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "0fdf8a01-00ed-4e40-8b8b-3e4ecfe37d45", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import importlib\n", - "import helpers.hgoogle_drive_api as hgodrapi\n", - "\n", - "importlib.reload(hgodrapi)" - ] - }, - { - "cell_type": "markdown", - "id": "f9733115-f65b-43fb-8b56-32be7588c617", - "metadata": {}, - "source": [ - "\n", - "## Get Credentials for your drive" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "0f3eb12a-bd7e-4846-a8f0-331ece997137", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "google_creds = hgodrapi.get_credentials()\n", - "print(google_creds)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "feb74dae-ff52-44ce-b698-4c04cc2bc8f3", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "service = hgodrapi.get_sheets_service(credentials=google_creds)\n", - "print(service)" - ] - }, - { - "cell_type": "markdown", - "id": "9e1c8840-c759-4bd6-a2c5-f30d94daf72b", - "metadata": {}, - "source": [ - "\n", - "## Get Tab/Sheet id of a particular google sheet" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "67fe7cc1-0f90-4b45-b93d-c6eaecd25028", - "metadata": {}, - "outputs": [], - "source": [ - "sheet_name = \"cleaned_profiles_1\"\n", - "url = \"https://docs.google.com/spreadsheets/d/1VRJQZ4kSoqAeOr9MkWcYbIcArNRyglTREaMg1WlZHGA/edit?gid=1687996260#gid=1687996260\"\n", - "sheet_id = \"1VRJQZ4kSoqAeOr9MkWcYbIcArNRyglTREaMg1WlZHGA\"\n", - "credentials = google_creds" - ] - }, - { - "cell_type": "markdown", - "id": "f18db947-8170-4cba-8799-dfe792e1c732", - "metadata": {}, - "source": [ - "\n", - "## Freeze Rows" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "232a1ee0-83d2-4449-a8c0-a8e8eca02fc5", - "metadata": {}, - "outputs": [], - "source": [ - "row_indices = [0, 1, 2]\n", - "hgodrapi.freeze_rows(\n", - " sheet_id=sheet_id, \n", - " row_indices=row_indices,\n", - " sheet_name = sheet_name,\n", - " credentials=credentials\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "dd9b9b7d-2dc6-416d-bd9c-a8039fadaba2", - "metadata": {}, - "source": [ - "\n", - "## Change the height of certin rows" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "50db6e3d-8d05-47ea-9ace-dc79ce131f37", - "metadata": {}, - "outputs": [], - "source": [ - "hgodrapi.set_row_height(sheet_id=sheet_id,height= 20, start_index = 0, end_index =2, sheet_name = sheet_name,credentials=google_creds)" - ] - }, - { - "cell_type": "markdown", - "id": "3569d048-d69e-4e4b-ab53-a93b6f4a41d1", - "metadata": {}, - "source": [ - "\n", - "## Read some nice data" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "e63bede3-2948-4a37-b444-36b4dba81c6d", - "metadata": {}, - "outputs": [], - "source": [ - "nice_data = hgodrapi.read_google_file(url, sheet_name, credentials=google_creds)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "59233081-ac03-4ac7-96b1-4de1b07fae75", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
NameTitleFirmLocationPersonal Investment FocusGeographic FocusTypical Deal SizePreferred StagesNotable Personal AI InvestmentsPotential Fit with KaizenfirstNamelastName
0Michael MoritzManaging PartnerSequoia CapitalMenlo ParkAI/MLGlobal$10MEarly to GrowthGoogleHigh'''MichaelMoritz
1Navid AlipourManaging PartnerAnalytics VenturesSan DiegoAI/MLSan Diego$5MSeedCureMetrixHigh''NavidAlipour
2Aaref HilalyPartnerBain Capital VenturesPalo AltoReal-time AnalyticsBurlingameSeedSeed/Early StageRubrikMediumAarefHilaly
3Aaron FleishmanPrincipalTola CapitalSeattle WAEnterprise AIPNW$5M-$20MSeries ADatabricksHighAaronFleishman
4Aaron JacobsonPartnerNew Enterprise AssociatesMenlo ParkMLOpsNorth AmericaSeries A$10M-$30MDatabricksHighAaronJacobson
\n", - "
" - ], - "text/plain": [ - " Name Title Firm Location \\\n", - "0 Michael Moritz Managing Partner Sequoia Capital Menlo Park \n", - "1 Navid Alipour Managing Partner Analytics Ventures San Diego \n", - "2 Aaref Hilaly Partner Bain Capital Ventures Palo Alto \n", - "3 Aaron Fleishman Principal Tola Capital Seattle WA \n", - "4 Aaron Jacobson Partner New Enterprise Associates Menlo Park \n", - "\n", - " Personal Investment Focus Geographic Focus Typical Deal Size \\\n", - "0 AI/ML Global $10M \n", - "1 AI/ML San Diego $5M \n", - "2 Real-time Analytics Burlingame Seed \n", - "3 Enterprise AI PNW $5M-$20M \n", - "4 MLOps North America Series A \n", - "\n", - " Preferred Stages Notable Personal AI Investments Potential Fit with Kaizen \\\n", - "0 Early to Growth Google High''' \n", - "1 Seed CureMetrix High'' \n", - "2 Seed/Early Stage Rubrik Medium \n", - "3 Series A Databricks High \n", - "4 $10M-$30M Databricks High \n", - "\n", - " firstName lastName \n", - "0 Michael Moritz \n", - "1 Navid Alipour \n", - "2 Aaref Hilaly \n", - "3 Aaron Fleishman \n", - "4 Aaron Jacobson " - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nice_data.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "30dcc791-cbdb-45f1-9298-a74e0a7babab", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(100, 12)" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nice_data.shape" - ] - }, - { - "cell_type": "markdown", - "id": "5c4cafb4-fe5f-4f6e-b594-759b199acb7e", - "metadata": {}, - "source": [ - "\n", - "## Write this nice data" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "3b1f4a89-cb96-417a-86f4-ebc513c18510", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Tab 'testing_tab' not found, creating a new tab with the name 'testing_tab'...\n" - ] - } - ], - "source": [ - "hgodrapi.write_to_google_sheet(nice_data, url, \"testing_tab\", credentials=google_creds)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "id": "7fb27b941602401d91542211134fc71a", + "metadata": {}, + "source": [ + "CONTENTS:\n", + "- [hgoogle_file_api.py](#hgoogle_file_api.py)\n", + " - [Get Credentials for your drive](#get-credentials-for-your-drive)\n", + " - [Get Tab/Sheet id of a particular google sheet](#get-tab/sheet-id-of-a-particular-google-sheet)\n", + " - [Freeze Rows](#freeze-rows)\n", + " - [Change the height of certin rows](#change-the-height-of-certin-rows)\n", + " - [Read some nice data](#read-some-nice-data)\n", + " - [Write this nice data](#write-this-nice-data)" + ] + }, + { + "cell_type": "markdown", + "id": "982ab891-de0a-47d5-946a-0f4fd3f16307", + "metadata": {}, + "source": [ + "\n", + "# hgoogle_file_api.py" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "6b997caf-4bfc-47bc-b7e1-584f02da328f", + "metadata": {}, + "outputs": [], + "source": [ + "#!sudo /bin/bash -c \"(source /venv/bin/activate; pip install --upgrade google-api-python-client)\"\n", + "# !sudo /bin/bash -c \"(source /venv/bin/activate; pip install --upgrade pip install oauth2client)\"\n", + "#!sudo /bin/bash -c \"(source /venv/bin/activate; pip install --upgrade gspread)\"" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "0fdf8a01-00ed-4e40-8b8b-3e4ecfe37d45", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import importlib\n", + "import helpers.hgoogle_drive_api as hgodrapi\n", + "\n", + "importlib.reload(hgodrapi)" + ] + }, + { + "cell_type": "markdown", + "id": "f9733115-f65b-43fb-8b56-32be7588c617", + "metadata": {}, + "source": [ + "\n", + "## Get Credentials for your drive" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "0f3eb12a-bd7e-4846-a8f0-331ece997137", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "google_creds = hgodrapi.get_credentials()\n", + "print(google_creds)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "feb74dae-ff52-44ce-b698-4c04cc2bc8f3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "service = hgodrapi.get_sheets_service(credentials=google_creds)\n", + "print(service)" + ] + }, + { + "cell_type": "markdown", + "id": "9e1c8840-c759-4bd6-a2c5-f30d94daf72b", + "metadata": {}, + "source": [ + "\n", + "## Get Tab/Sheet id of a particular google sheet" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "67fe7cc1-0f90-4b45-b93d-c6eaecd25028", + "metadata": {}, + "outputs": [], + "source": [ + "sheet_name = \"cleaned_profiles_1\"\n", + "url = \"https://docs.google.com/spreadsheets/d/1VRJQZ4kSoqAeOr9MkWcYbIcArNRyglTREaMg1WlZHGA/edit?gid=1687996260#gid=1687996260\"\n", + "sheet_id = \"1VRJQZ4kSoqAeOr9MkWcYbIcArNRyglTREaMg1WlZHGA\"\n", + "credentials = google_creds" + ] + }, + { + "cell_type": "markdown", + "id": "f18db947-8170-4cba-8799-dfe792e1c732", + "metadata": {}, + "source": [ + "\n", + "## Freeze Rows" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "232a1ee0-83d2-4449-a8c0-a8e8eca02fc5", + "metadata": {}, + "outputs": [], + "source": [ + "row_indices = [0, 1, 2]\n", + "hgodrapi.freeze_rows(\n", + " sheet_id=sheet_id,\n", + " row_indices=row_indices,\n", + " sheet_name=sheet_name,\n", + " credentials=credentials,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "dd9b9b7d-2dc6-416d-bd9c-a8039fadaba2", + "metadata": {}, + "source": [ + "\n", + "## Change the height of certin rows" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "50db6e3d-8d05-47ea-9ace-dc79ce131f37", + "metadata": {}, + "outputs": [], + "source": [ + "hgodrapi.set_row_height(\n", + " sheet_id=sheet_id,\n", + " height=20,\n", + " start_index=0,\n", + " end_index=2,\n", + " sheet_name=sheet_name,\n", + " credentials=google_creds,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "3569d048-d69e-4e4b-ab53-a93b6f4a41d1", + "metadata": {}, + "source": [ + "\n", + "## Read some nice data" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "e63bede3-2948-4a37-b444-36b4dba81c6d", + "metadata": {}, + "outputs": [], + "source": [ + "nice_data = hgodrapi.read_google_file(url, sheet_name, credentials=google_creds)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "59233081-ac03-4ac7-96b1-4de1b07fae75", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
NameTitleFirmLocationPersonal Investment FocusGeographic FocusTypical Deal SizePreferred StagesNotable Personal AI InvestmentsPotential Fit with KaizenfirstNamelastName
0Michael MoritzManaging PartnerSequoia CapitalMenlo ParkAI/MLGlobal$10MEarly to GrowthGoogleHigh'''MichaelMoritz
1Navid AlipourManaging PartnerAnalytics VenturesSan DiegoAI/MLSan Diego$5MSeedCureMetrixHigh''NavidAlipour
2Aaref HilalyPartnerBain Capital VenturesPalo AltoReal-time AnalyticsBurlingameSeedSeed/Early StageRubrikMediumAarefHilaly
3Aaron FleishmanPrincipalTola CapitalSeattle WAEnterprise AIPNW$5M-$20MSeries ADatabricksHighAaronFleishman
4Aaron JacobsonPartnerNew Enterprise AssociatesMenlo ParkMLOpsNorth AmericaSeries A$10M-$30MDatabricksHighAaronJacobson
\n", + "
" + ], + "text/plain": [ + " Name Title Firm Location \\\n", + "0 Michael Moritz Managing Partner Sequoia Capital Menlo Park \n", + "1 Navid Alipour Managing Partner Analytics Ventures San Diego \n", + "2 Aaref Hilaly Partner Bain Capital Ventures Palo Alto \n", + "3 Aaron Fleishman Principal Tola Capital Seattle WA \n", + "4 Aaron Jacobson Partner New Enterprise Associates Menlo Park \n", + "\n", + " Personal Investment Focus Geographic Focus Typical Deal Size \\\n", + "0 AI/ML Global $10M \n", + "1 AI/ML San Diego $5M \n", + "2 Real-time Analytics Burlingame Seed \n", + "3 Enterprise AI PNW $5M-$20M \n", + "4 MLOps North America Series A \n", + "\n", + " Preferred Stages Notable Personal AI Investments Potential Fit with Kaizen \\\n", + "0 Early to Growth Google High''' \n", + "1 Seed CureMetrix High'' \n", + "2 Seed/Early Stage Rubrik Medium \n", + "3 Series A Databricks High \n", + "4 $10M-$30M Databricks High \n", + "\n", + " firstName lastName \n", + "0 Michael Moritz \n", + "1 Navid Alipour \n", + "2 Aaref Hilaly \n", + "3 Aaron Fleishman \n", + "4 Aaron Jacobson " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nice_data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "30dcc791-cbdb-45f1-9298-a74e0a7babab", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(100, 12)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nice_data.shape" + ] + }, + { + "cell_type": "markdown", + "id": "5c4cafb4-fe5f-4f6e-b594-759b199acb7e", + "metadata": {}, + "source": [ + "\n", + "## Write this nice data" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "3b1f4a89-cb96-417a-86f4-ebc513c18510", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Tab 'testing_tab' not found, creating a new tab with the name 'testing_tab'...\n" + ] + } + ], + "source": [ + "hgodrapi.write_to_google_sheet(\n", + " nice_data, url, \"testing_tab\", credentials=google_creds\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/helpers/notebooks/Master.how_to_use_hgoogle_drive_api.py b/helpers/notebooks/Master.how_to_use_hgoogle_drive_api.py index 4bba188f6..0da4c7073 100644 --- a/helpers/notebooks/Master.how_to_use_hgoogle_drive_api.py +++ b/helpers/notebooks/Master.how_to_use_hgoogle_drive_api.py @@ -52,10 +52,10 @@ # %% row_indices = [0, 1, 2] hgodrapi.freeze_rows( - sheet_id=sheet_id, + sheet_id=sheet_id, row_indices=row_indices, - sheet_name = sheet_name, - credentials=credentials + sheet_name=sheet_name, + credentials=credentials, ) # %% [markdown] @@ -63,7 +63,14 @@ # ## Change the height of certin rows # %% -hgodrapi.set_row_height(sheet_id=sheet_id,height= 20, start_index = 0, end_index =2, sheet_name = sheet_name,credentials=google_creds) +hgodrapi.set_row_height( + sheet_id=sheet_id, + height=20, + start_index=0, + end_index=2, + sheet_name=sheet_name, + credentials=google_creds, +) # %% [markdown] # @@ -83,6 +90,6 @@ # ## Write this nice data # %% -hgodrapi.write_to_google_sheet(nice_data, url, "testing_tab", credentials=google_creds) - - +hgodrapi.write_to_google_sheet( + nice_data, url, "testing_tab", credentials=google_creds +) diff --git a/helpers/notebooks/PartTask2291_Playback_approach_for_unit_testing.ipynb b/helpers/notebooks/PartTask2291_Playback_approach_for_unit_testing.ipynb index 919d29165..d38079e6b 100644 --- a/helpers/notebooks/PartTask2291_Playback_approach_for_unit_testing.ipynb +++ b/helpers/notebooks/PartTask2291_Playback_approach_for_unit_testing.ipynb @@ -545,7 +545,6 @@ "outputs": [], "source": [ "class Playback:\n", - "\n", " # def __init__(self, file_name, mode, *args, **kwargs):\n", " # self.args = args\n", " # self.kwargs = kwargs\n", @@ -603,7 +602,6 @@ "outputs": [], "source": [ "class Playback:\n", - "\n", " # def __init__(self, file_name, mode, *args, **kwargs):\n", " # self.args = args\n", " # self.kwargs = kwargs\n", diff --git a/helpers/notebooks/PartTask2291_Playback_approach_for_unit_testing.py b/helpers/notebooks/PartTask2291_Playback_approach_for_unit_testing.py index c5a287967..eb3750fff 100644 --- a/helpers/notebooks/PartTask2291_Playback_approach_for_unit_testing.py +++ b/helpers/notebooks/PartTask2291_Playback_approach_for_unit_testing.py @@ -49,7 +49,6 @@ hprint.config_notebook() # %% -import pandas as pd data = { "Product": ["Desktop Computer", "Tablet", "iPhone", "Laptop"], @@ -128,7 +127,6 @@ def F(a, b): # %% class Playback: - # def __init__(self, file_name, mode, *args, **kwargs): # self.args = args # self.kwargs = kwargs @@ -176,7 +174,6 @@ def F(a: int, b: int): # %% class Playback: - # def __init__(self, file_name, mode, *args, **kwargs): # self.args = args # self.kwargs = kwargs diff --git a/helpers/notebooks/cache.py b/helpers/notebooks/cache.py index d13fa13f7..f98bfb61a 100644 --- a/helpers/notebooks/cache.py +++ b/helpers/notebooks/cache.py @@ -40,6 +40,7 @@ # %% [markdown] pycharm={"name": "#%% md\n"} # # Define computation function + # %% def func(a, b): # hello diff --git a/helpers/notebooks/gallery_s3.ipynb b/helpers/notebooks/gallery_s3.ipynb index 19e343fec..36df5c337 100644 --- a/helpers/notebooks/gallery_s3.ipynb +++ b/helpers/notebooks/gallery_s3.ipynb @@ -43,7 +43,6 @@ }, "outputs": [], "source": [ - "\n", "import helpers.hs3 as hs3" ] }, diff --git a/helpers/notebooks/hopenai_tutorial.py b/helpers/notebooks/hopenai_tutorial.py index 5a40ef9c9..e607c8637 100644 --- a/helpers/notebooks/hopenai_tutorial.py +++ b/helpers/notebooks/hopenai_tutorial.py @@ -67,10 +67,10 @@ # %% # Normalize the nested JSON -df = pd.json_normalize(val, sep='_') +df = pd.json_normalize(val, sep="_") df # View the resulting DataFrame -#print(df.T) # Transpose just for readable vertical inspection +# print(df.T) # Transpose just for readable vertical inspection # %% df.iloc[0].T @@ -96,7 +96,7 @@ # %% -pd.to_numeric(df["pricing_request"], errors='coerce').notna() +pd.to_numeric(df["pricing_request"], errors="coerce").notna() # %% df["pricing_completion"] @@ -105,7 +105,9 @@ df.sort_values("pricing_prompt")[col_names] # %% -df[["pricing_prompt", "pricing_completion"]].plot.scatter(x="pricing_prompt", y="pricing_completion") +df[["pricing_prompt", "pricing_completion"]].plot.scatter( + x="pricing_prompt", y="pricing_completion" +) # %% df["price_ratio"] = df["pricing_completion"] / df["pricing_prompt"] @@ -113,4 +115,4 @@ # %% # %% -#df["total_price"] = +# df["total_price"] = diff --git a/helpers/old/tunnels.py b/helpers/old/tunnels.py index 31eadfc87..cef11e302 100644 --- a/helpers/old/tunnels.py +++ b/helpers/old/tunnels.py @@ -41,7 +41,7 @@ def tunnel_info_to_string(tunnel_info: list) -> str: def parse_service( - service: Tuple[str, str, int, int] + service: Tuple[str, str, int, int], ) -> Dict[str, Union[str, int]]: hdbg.dassert_eq(len(service), 4, "service=%s", service) service_name, server, local_port, remote_port = service @@ -258,6 +258,7 @@ def kill_all_tunnel_processes() -> None: """ Kill all the processes that have `ssh -i ...:localhost:...". """ + # cmd = "ps ax | grep 'ssh -i' | grep localhost: | grep -v grep" def _keep_line(line: str) -> bool: keep = ("ssh -i" in line) and (":localhost:" in line) diff --git a/helpers/old/user_credentials.py b/helpers/old/user_credentials.py index dab17fee6..5faded15d 100755 --- a/helpers/old/user_credentials.py +++ b/helpers/old/user_credentials.py @@ -26,7 +26,9 @@ def get_dev_server_ip() -> str: """ env_var_name = "" if env_var_name not in os.environ: - _LOG.error("Can't find '%s': re-run dev_scripts/setenv.sh?", env_var_name) + _LOG.error( + "Can't find '%s': re-run dev_scripts/setenv.sh?", env_var_name + ) raise RuntimeError dev_server = os.environ[env_var_name] return dev_server @@ -183,7 +185,8 @@ def get_credentials() -> Dict[str, Any]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--user", action="store", default=None, help="Impersonate a user" diff --git a/helpers/repo_config_utils.py b/helpers/repo_config_utils.py index 9edc92f87..89b62c6e9 100644 --- a/helpers/repo_config_utils.py +++ b/helpers/repo_config_utils.py @@ -105,7 +105,6 @@ def _get_env_var( class RepoConfig: - def __init__(self, data: Dict) -> None: """ Set the data to be used by the module. diff --git a/helpers/stage_linked_file.py b/helpers/stage_linked_file.py index 43d83e881..e03db6dc8 100644 --- a/helpers/stage_linked_file.py +++ b/helpers/stage_linked_file.py @@ -60,7 +60,9 @@ def main(): parser = argparse.ArgumentParser( description="Stage symbolic links for modification." ) - parser.add_argument("--dst_dir", required=True, help="Destination directory.") + parser.add_argument( + "--dst_dir", required=True, help="Destination directory." + ) args = parser.parse_args() symlinks = find_symlinks(args.dst_dir) diff --git a/helpers/telegram_notify/get_chat_id.py b/helpers/telegram_notify/get_chat_id.py index ed5202d7a..e90c3968d 100644 --- a/helpers/telegram_notify/get_chat_id.py +++ b/helpers/telegram_notify/get_chat_id.py @@ -56,7 +56,8 @@ def send_chat_id(token: str, username: str) -> str: def _main() -> None: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--username", required=True, action="store", type=str) parser.add_argument("--token", required=False, action="store", type=str) diff --git a/helpers/test/test_hcache.py b/helpers/test/test_hcache.py index d8a5eaa38..bf4124c6e 100644 --- a/helpers/test/test_hcache.py +++ b/helpers/test/test_hcache.py @@ -258,7 +258,9 @@ def test_with_caching4(self) -> None: - Cache different values """ # Use only memory cache. - f, cf = self._get_f_cf_functions(use_mem_cache=True, use_disk_cache=False) + f, cf = self._get_f_cf_functions( + use_mem_cache=True, use_disk_cache=False + ) # 1) Execute and verify that it is executed since not cached. _LOG.debug("\n%s", hprint.frame("Execute the 1st time")) self._execute_and_check_state(f, cf, 3, 4, exp_cf_state="no_cache") @@ -276,7 +278,9 @@ def test_with_caching5(self) -> None: - Cache different values """ # Use only disk cache. - f, cf = self._get_f_cf_functions(use_mem_cache=False, use_disk_cache=True) + f, cf = self._get_f_cf_functions( + use_mem_cache=False, use_disk_cache=True + ) # 1) Verify that it is executed since there is no cache. _LOG.debug("\n%s", hprint.frame("Execute the 1st time")) self._execute_and_check_state(f, cf, 3, 4, exp_cf_state="no_cache") @@ -299,7 +303,9 @@ def test_with_caching_mem_reset(self) -> None: - Check that the cached function is recomputed """ # Use only memory cache. - f, cf = self._get_f_cf_functions(use_mem_cache=True, use_disk_cache=False) + f, cf = self._get_f_cf_functions( + use_mem_cache=True, use_disk_cache=False + ) # 1) Verify that it is executed, since it's not cached. _LOG.debug("\n%s", hprint.frame("Execute the 1st time")) self._execute_and_check_state(f, cf, 3, 4, exp_cf_state="no_cache") @@ -318,7 +324,9 @@ def test_with_caching_disk_reset(self) -> None: Same as `test_with_caching_mem_reset()` but using the disk cache. """ # Use only disk cache. - f, cf = self._get_f_cf_functions(use_mem_cache=False, use_disk_cache=True) + f, cf = self._get_f_cf_functions( + use_mem_cache=False, use_disk_cache=True + ) # 1) Verify that it is executed, since it's not cached. _LOG.debug("\n%s", hprint.frame("Execute the 1st time")) self._execute_and_check_state(f, cf, 3, 4, exp_cf_state="no_cache") diff --git a/helpers/test/test_hcache_simple.py b/helpers/test/test_hcache_simple.py index 1a4f174cf..89647923e 100644 --- a/helpers/test/test_hcache_simple.py +++ b/helpers/test/test_hcache_simple.py @@ -196,7 +196,6 @@ def tear_down_test(self) -> None: class Test_get_cache(BaseCacheTest): - def test1(self) -> None: """ Verify that get_cache returns a cache with the expected key and value. @@ -216,7 +215,6 @@ def test1(self) -> None: class Test_flush_cache_to_disk(BaseCacheTest): - def test1(self) -> None: """ Verify that flushing creates a cache file on disk. @@ -258,7 +256,6 @@ def test2(self) -> None: class Test_reset_mem_cache(BaseCacheTest): - def test1(self) -> None: """ Verify that the cache is empty after `reset_mem_cache` is called. @@ -279,7 +276,6 @@ def test1(self) -> None: class Test_force_cache_from_disk(BaseCacheTest): - def test1(self) -> None: """ Verify that the memory cache is empty after a reset. @@ -310,7 +306,9 @@ def test2(self) -> None: full_cache: Dict[str, Any] = hcacsimp.get_cache("_cached_function") # Assert that the key is restored in the in-memory cache. self.assertIn( - "(7,)", full_cache, "After forcing, disk key should appear in memory." + "(7,)", + full_cache, + "After forcing, disk key should appear in memory.", ) @@ -320,7 +318,6 @@ def test2(self) -> None: class Test_get_cache_perf(BaseCacheTest): - def test1(self) -> None: """ Verify that performance tracking records hits and misses correctly. @@ -354,7 +351,6 @@ def test2(self) -> None: class Test_set_cache_property(BaseCacheTest): - def test1(self) -> None: """ Verify that setting a valid cache property works and can be retrieved. @@ -410,7 +406,9 @@ def test4(self) -> None: hcacsimp.set_cache_property( "user", "_cached_function", "force_refresh", True ) - prop_str: str = hcacsimp.cache_property_to_str("user", "_cached_function") + prop_str: str = hcacsimp.cache_property_to_str( + "user", "_cached_function" + ) # Check output. self.assertIn("force_refresh: True", prop_str) @@ -421,7 +419,6 @@ def test4(self) -> None: class Test_get_cache_func_names(BaseCacheTest): - def test1(self) -> None: """ Verify that memory cache function names include `_cached_function`. @@ -473,7 +470,6 @@ def test3(self) -> None: class Test_cache_stats_to_str(BaseCacheTest): - def test1(self) -> None: """ Verify that cache_stats_to_str returns a DataFrame with 'memory' and @@ -497,7 +493,6 @@ def test1(self) -> None: class Test__kwarg_func(BaseCacheTest): - def test1(self) -> None: """ Test that verifies keyword arguments are handled correctly by the @@ -516,7 +511,6 @@ def test1(self) -> None: class Test__multi_arg_func(BaseCacheTest): - def test1(self) -> None: """ Verify that the cache for _multi_arg_func contains the correct key. @@ -534,7 +528,6 @@ def test1(self) -> None: class Test__cached_pickle_function(BaseCacheTest): - def test1(self) -> None: """ Ensure that _cached_pickle_function returns the correct value and disk @@ -560,7 +553,6 @@ def test1(self) -> None: class Test__refreshable_function(BaseCacheTest): - def test1(self) -> None: """ Verify that `_refreshable_function` is called only once initially. diff --git a/helpers/test/test_hcfile.py b/helpers/test/test_hcfile.py index 87d70f942..25ec97e94 100644 --- a/helpers/test/test_hcfile.py +++ b/helpers/test/test_hcfile.py @@ -3,7 +3,6 @@ from typing import Any, List import helpers.hio as hio -import helpers.hmarkdown as hmarkdo import helpers.hprint as hprint import helpers.hunit_test as hunitest import helpers.hcfile as hcfile @@ -45,7 +44,6 @@ def _create_cfile(self_: Any, cfile_content: List[str]) -> str: class Test_parse_cfile1(hunitest.TestCase): - def helper(self, cfile_content: str, expected: str) -> None: """ Helper function to test parsing a cfile. @@ -125,7 +123,6 @@ def test_invalid_entries(self) -> None: class Test_inject_todos_from_cfile1(hunitest.TestCase): - def test1(self) -> None: """ Test injecting TODOs from a cfile into a Python file. diff --git a/helpers/test/test_hdataframe.py b/helpers/test/test_hdataframe.py index f93dbf851..4793fb101 100644 --- a/helpers/test/test_hdataframe.py +++ b/helpers/test/test_hdataframe.py @@ -25,7 +25,6 @@ class Test_filter_data_by_values1(hunitest.TestCase): - def test_conjunction1(self) -> None: data = pd.DataFrame([[1, 2, 3], [4, 5, 6]]) data = data.add_prefix("col_") @@ -66,7 +65,6 @@ def test_disjunction1(self) -> None: class Test_filter_data_by_comparison(hunitest.TestCase): - def test_conjunction1(self) -> None: data = pd.DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]]) data = data.add_prefix("col_") @@ -140,7 +138,6 @@ def test1(self) -> None: class Test_apply_nan_mode(hunitest.TestCase): - def test1(self) -> None: """ Test for `mode=leave_unchanged`. @@ -211,7 +208,6 @@ def _get_series_with_nans(seed: int) -> pd.Series: class Test_compute_points_per_year_for_given_freq(hunitest.TestCase): - def test1(self) -> None: actual = hdatafr.compute_points_per_year_for_given_freq("T") np.testing.assert_equal(actual, 525780.125) @@ -247,7 +243,6 @@ def test7(self) -> None: class TestRemoveDuplicates(hunitest.TestCase): - def test_remove_duplicates1(self) -> None: test_data = { "dummy_value_1": [1, 2, 1], diff --git a/helpers/test/test_hdatetime.py b/helpers/test/test_hdatetime.py index b7cb14421..1a5946193 100644 --- a/helpers/test/test_hdatetime.py +++ b/helpers/test/test_hdatetime.py @@ -244,7 +244,12 @@ def test_dassert_compatible_timestamp_assert2(self) -> None: Test a pairs of non-compatible datetimes making sure the assertion is raised. """ - for datetime1 in [_PD_TS_NAIVE, _DT_DT_NAIVE, _PD_TS_NAIVE, _DT_DT_NAIVE]: + for datetime1 in [ + _PD_TS_NAIVE, + _DT_DT_NAIVE, + _PD_TS_NAIVE, + _DT_DT_NAIVE, + ]: for datetime2 in [_PD_TS_UTC, _PD_TS_ET, _DT_DT_UTC, _DT_DT_ET]: with self.assertRaises(AssertionError): hdateti.dassert_tz_compatible(datetime1, datetime2) @@ -831,8 +836,8 @@ def test4(self) -> None: """ # Check. self.assert_equal(act, exp, fuzzy_match=True) - - + + # ############################################################################# # Test_dassert_timestamp_lt # ############################################################################# diff --git a/helpers/test/test_hdocker.py b/helpers/test/test_hdocker.py index e6f86eafd..a199bbb7c 100644 --- a/helpers/test/test_hdocker.py +++ b/helpers/test/test_hdocker.py @@ -22,7 +22,6 @@ class Test_replace_shared_root_path1(hunitest.TestCase): - def test1(self) -> None: """ Test replacing shared root path. @@ -84,7 +83,6 @@ def test2(self) -> None: class Test_convert_to_docker_path1(hunitest.TestCase): - @staticmethod def convert_caller_to_callee_docker_path( in_file_path: str, @@ -153,10 +151,12 @@ def test1(self) -> None: check_if_exists = False # - Prepare outputs. helpers_root_path = hgit.find_helpers_root() - exp_docker_file_path = os.path.join(helpers_root_path, + exp_docker_file_path = os.path.join( + helpers_root_path, "helpers/test/outcomes", "Test_convert_to_docker_path1.test1/input", - "tmp.llm_transform.in.txt") + "tmp.llm_transform.in.txt", + ) exp_mount = "type=bind,source=/app,target=/app" self.helper( in_file_path, @@ -183,10 +183,12 @@ def test2(self) -> None: check_if_exists = True # - Prepare outputs. helpers_root_path = hgit.find_helpers_root() - exp_docker_file_path = os.path.join(helpers_root_path, + exp_docker_file_path = os.path.join( + helpers_root_path, "helpers/test/outcomes", "Test_convert_to_docker_path1.test2/input", - "tmp.input.md") + "tmp.input.md", + ) exp_mount = "type=bind,source=/app,target=/app" self.helper( in_file_path, @@ -288,7 +290,6 @@ def test2(self) -> None: class Test_parse_pandoc_arguments1(hunitest.TestCase): - def test1(self) -> None: # Prepare inputs. cmd = r""" @@ -467,7 +468,6 @@ def test1(self) -> None: reason="Disabled because of CmampTask10710", ) class Test_run_markdown_toc1(hunitest.TestCase): - def run_markdown_toc(self, txt: str, exp: str) -> None: """ Test running the `markdown-toc` command in a Docker container. @@ -535,7 +535,6 @@ def test1(self) -> None: reason="Disabled because of CmampTask10710", ) class Test_dockerized_latex1(hunitest.TestCase): - def create_input_file(self) -> Tuple[str, str]: txt = r""" \documentclass{article} @@ -632,7 +631,6 @@ def test_cmd_line1(self) -> None: reason="Disabled because of CmampTask10710", ) class Test_dockerized_tikz_to_bitmap1(hunitest.TestCase): - def create_input_file(self) -> Tuple[str, str]: txt = r""" \documentclass[tikz, border=10pt]{standalone} @@ -712,7 +710,6 @@ def test_command_line1(self) -> None: reason="Disabled because of CmampTask10710", ) class Test_dockerized_graphviz1(hunitest.TestCase): - def create_input_file(self) -> Tuple[str, str]: txt = r""" digraph { diff --git a/helpers/test/test_henv.py b/helpers/test/test_henv.py index aca2d1407..7eeea1af8 100644 --- a/helpers/test/test_henv.py +++ b/helpers/test/test_henv.py @@ -12,7 +12,6 @@ class Test_env1(hunitest.TestCase): - def test_get_system_signature1(self) -> None: txt = henv.get_system_signature() _LOG.debug(txt) @@ -27,4 +26,4 @@ def test_has_not_module1(self) -> None: """ Check that the function returns false for the non-existing package. """ - self.assertFalse(henv.has_module("no_such_module")) \ No newline at end of file + self.assertFalse(henv.has_module("no_such_module")) diff --git a/helpers/test/test_hgit.py b/helpers/test/test_hgit.py index dd5a8f8a7..14af67958 100644 --- a/helpers/test/test_hgit.py +++ b/helpers/test/test_hgit.py @@ -23,7 +23,6 @@ class Test_git_submodule1(hunitest.TestCase): - def test_get_client_root1(self) -> None: act = hgit.get_client_root(super_module=True) _LOG.debug("act=%s", act) @@ -125,7 +124,6 @@ def _helper_group_hashes( class Test_git_repo_name1(hunitest.TestCase): - def test_parse_github_repo_name1(self) -> None: repo_name = "git@github.com:alphamatic/amp" host_name, repo_name = hgit._parse_github_repo_name(repo_name) @@ -180,7 +178,6 @@ def test_get_repo_full_name_from_client2(self) -> None: # Outside CK infra, the following class hangs, so we skip it. @pytest.mark.requires_ck_infra class Test_git_path1(hunitest.TestCase): - @pytest.mark.skipif( not hgit.is_in_amp_as_supermodule(), reason="Run only in amp as super-module", @@ -194,7 +191,8 @@ def test_get_path_from_git_root1(self) -> None: self.assert_equal(act, exp) @pytest.mark.skipif( - not hgit.is_in_amp_as_submodule(), reason="Run only in amp as sub-module" + not hgit.is_in_amp_as_submodule(), + reason="Run only in amp as sub-module", ) def test_get_path_from_git_root2(self) -> None: file_name = "/app/amp/helpers/test/test_hgit.py" @@ -291,7 +289,6 @@ def test_git_log1(self) -> None: # Outside CK infra, the following class hangs, so we skip it. @pytest.mark.requires_ck_infra class Test_find_docker_file1(hunitest.TestCase): - def test1(self) -> None: """ Test for a file `amp/helpers/test/test_hgit.py` that is not from Docker @@ -389,7 +386,6 @@ def test5(self) -> None: class Test_extract_gh_issue_number_from_branch(hunitest.TestCase): - def test_extract_gh_issue_number_from_branch1(self) -> None: """ Tests extraction from a branch name with a specific format. @@ -460,7 +456,7 @@ def set_up_test(self) -> None: self.submodule_dir = os.path.join(self.repo_dir, "amp") hio.create_dir(self.submodule_dir, incremental=False) submodule_git_file = os.path.join(self.submodule_dir, ".git") - txt = f"gitdir: ../.git/modules/amp" + txt = "gitdir: ../.git/modules/amp" hio.to_file(submodule_git_file, txt) submodule_git_file_dir = os.path.join( self.repo_dir, ".git", "modules", "amp" @@ -470,7 +466,7 @@ def set_up_test(self) -> None: self.subsubmodule_dir = os.path.join(self.submodule_dir, "helpers_root") hio.create_dir(self.subsubmodule_dir, incremental=False) subsubmodule_git_file = os.path.join(self.subsubmodule_dir, ".git") - txt = f"gitdir: ../../.git/modules/amp/modules/helpers_root" + txt = "gitdir: ../../.git/modules/amp/modules/helpers_root" hio.to_file(subsubmodule_git_file, txt) subsubmodule_git_file_dir = os.path.join( self.repo_dir, ".git", "modules", "amp", "modules", "helpers_root" @@ -552,7 +548,7 @@ def set_up_test(self) -> None: self.submodule_dir = os.path.join(self.repo_dir, "helpers_root") hio.create_dir(self.submodule_dir, incremental=False) submodule_git_file = os.path.join(self.submodule_dir, ".git") - txt = f"gitdir: ../.git/modules/helpers_root" + txt = "gitdir: ../.git/modules/helpers_root" hio.to_file(submodule_git_file, txt) submodule_git_file_dir = os.path.join( self.repo_dir, ".git", "modules", "helpers_root" @@ -737,9 +733,10 @@ def test1(self) -> None: Check that the error is raised when the caller is in a directory that is not either a git repo or a submodule. """ - with hsystem.cd(self.arbitrary_dir), self.assertRaises( - AssertionError - ) as cm: + with ( + hsystem.cd(self.arbitrary_dir), + self.assertRaises(AssertionError) as cm, + ): _ = hgit.find_git_root(".") act = str(cm.exception) exp = """ @@ -759,7 +756,7 @@ def test2(self) -> None: with hsystem.cd(self.repo_dir), self.assertRaises(AssertionError) as cm: _ = hgit.find_git_root(".") act = str(cm.exception) - exp = f""" + exp = """ * Failed assertion * '/' != diff --git a/helpers/test/test_hintrospection.py b/helpers/test/test_hintrospection.py index e9a3d92b4..4c3bc54ad 100644 --- a/helpers/test/test_hintrospection.py +++ b/helpers/test/test_hintrospection.py @@ -89,9 +89,7 @@ def test_lambda1(self) -> None: # Local lambda. lambda_ = lambda: 0 func = lambda_ - exp_str = ( - r". at 0x>" - ) + exp_str = r". at 0x>" # A lambda is not bound to an object. exp_bound = False exp_lambda = True @@ -102,9 +100,7 @@ def test_lambda1(self) -> None: def test_lambda2(self) -> None: lambda_ = lambda x: x func = lambda_ - exp_str = ( - r". at 0x>" - ) + exp_str = r". at 0x>" # A lambda is not bound to an object. exp_bound = False exp_lambda = True @@ -365,7 +361,11 @@ def test1(self) -> None: # is a sub-repo or a super-repo: # helpers.test.test_hintrospection.dummy_function # helpers_root.helpers.test.test_hintrospection.dummy_function - # - act = re.sub(r"helpers_root\.helpers\.", "helpers.", act, flags=re.MULTILINE) - exp = re.sub(r"helpers_root\.helpers\.", "helpers.", exp, flags=re.MULTILINE) + # + act = re.sub( + r"helpers_root\.helpers\.", "helpers.", act, flags=re.MULTILINE + ) + exp = re.sub( + r"helpers_root\.helpers\.", "helpers.", exp, flags=re.MULTILINE + ) self.assert_equal(act, exp, purify_text=True) diff --git a/helpers/test/test_hio.py b/helpers/test/test_hio.py index 09b0a78dc..7a4e18549 100644 --- a/helpers/test/test_hio.py +++ b/helpers/test/test_hio.py @@ -18,7 +18,10 @@ class Test_rename_file_if_exists(hunitest.TestCase): """ def check_file( - self, file_to_rename: str, before_extension: bool, expected_file_name: str + self, + file_to_rename: str, + before_extension: bool, + expected_file_name: str, ) -> None: """ Check that file is renamed correctly. @@ -75,7 +78,9 @@ def test1(self) -> None: pattern = "*" only_files = True use_relative_paths = True - all_files = hio.listdir(dir_name, pattern, only_files, use_relative_paths) + all_files = hio.listdir( + dir_name, pattern, only_files, use_relative_paths + ) self.assertGreater(len(all_files), 0) # Check that there are more files than Python files. exclude_paired_jupytext = False diff --git a/helpers/test/test_hmarkdown.py b/helpers/test/test_hmarkdown.py index 66126bf26..55598addd 100644 --- a/helpers/test/test_hmarkdown.py +++ b/helpers/test/test_hmarkdown.py @@ -104,7 +104,6 @@ def get_header_list5() -> hmarkdo.HeaderList: class Test_header_list_to_vim_cfile1(hunitest.TestCase): - def test_get_header_list1(self) -> None: # Prepare inputs. markdown_file = "test.py" @@ -132,7 +131,6 @@ def test_get_header_list1(self) -> None: class Test_header_list_to_markdown1(hunitest.TestCase): - def test_mode_list1(self) -> None: # Prepare inputs. headers = get_header_list1() @@ -180,7 +178,6 @@ def test_mode_headers1(self) -> None: class Test_is_markdown_line_separator1(hunitest.TestCase): - def test_valid_separator1(self) -> None: # Prepare inputs. line = "-----------------------" @@ -437,7 +434,6 @@ def _get_markdown_example5() -> hmarkdo.HeaderList: class Test_extract_section_from_markdown1(hunitest.TestCase): - # TODO(gp): This doesn't seem correct. def test1(self) -> None: # Prepare inputs. @@ -511,7 +507,6 @@ def test_no_header(self) -> None: class Test_extract_headers_from_markdown1(hunitest.TestCase): - def test_multiple_headers(self) -> None: # Prepare inputs. content = _get_markdown_example1() @@ -549,7 +544,6 @@ def test_no_headers(self) -> None: class Test_remove_end_of_line_periods1(hunitest.TestCase): - def test_standard_case(self) -> None: txt = "Hello.\nWorld.\nThis is a test." act = hmarkdo.remove_end_of_line_periods(txt) @@ -587,7 +581,6 @@ def test_leading_and_trailing_periods(self) -> None: class Test_process_code_block1(hunitest.TestCase): - def process_code_block(self, txt: str) -> str: out: List[str] = [] in_code_block = False @@ -620,7 +613,6 @@ def test1(self) -> None: class Test_process_lines1(hunitest.TestCase): - # TODO(gp): This doesn't seem correct. def test1(self) -> None: in_dir_name = self.get_input_dir() @@ -694,7 +686,6 @@ def _test_full_navigation_flow(self_: Any, txt: str) -> None: class Test_selected_navigation_to_str1(hunitest.TestCase): - def test1(self) -> None: """ Create navigation bar from Markdown text `_get_markdown_example4()`. @@ -746,7 +737,6 @@ def test2(self) -> None: class Test_selected_navigation_to_str2(hunitest.TestCase): - def test1(self) -> None: """ Create navigation bar from Markdown text `_get_markdown_example5()`. @@ -792,7 +782,6 @@ def test2(self) -> None: class Test_bold_first_level_bullets1(hunitest.TestCase): - def test1(self) -> None: """ Test basic first-level bullet bolding. @@ -904,7 +893,6 @@ def _test_bold_first_level_bullets(self, text: str, expected: str) -> None: class Test_md_clean_up1(hunitest.TestCase): - def test1(self) -> None: # Prepare inputs. txt = r""" @@ -972,7 +960,6 @@ def test1(self) -> None: class Test_modify_header_level1(hunitest.TestCase): - def test1(self) -> None: """ Test the inputs to increase headings. @@ -1172,7 +1159,6 @@ def test8(self) -> None: class Test_format_headers1(hunitest.TestCase): - def test1(self) -> None: """ Test the inputs to check the basic formatting of headings. @@ -1289,7 +1275,6 @@ def _helper_process( class Test_remove_code_delimiters1(hunitest.TestCase): - def test1(self) -> None: """ Test a basic example. @@ -1448,7 +1433,6 @@ def no_start_python(): class Test_sanity_check_header_list1(hunitest.TestCase): - def test1(self) -> None: """ Test that the header list with valid level increase is accepted. @@ -1535,7 +1519,6 @@ def get_header_list6() -> hmarkdo.HeaderList: class Test_convert_header_list_into_guidelines1(hunitest.TestCase): - def test1(self) -> None: """ Test converting a header list into guidelines. @@ -1565,7 +1548,6 @@ def test1(self) -> None: class Test_extract_rules1(hunitest.TestCase): - def helper(self, selection_rules: List[str], exp: str) -> None: """ Test extracting rules from a markdown file. @@ -1631,7 +1613,6 @@ def test4(self) -> None: class Test_parse_rules_from_txt1(hunitest.TestCase): - def helper(self, text: str, expected: str) -> None: # Prepare inputs. text = hprint.dedent(text) @@ -1743,7 +1724,6 @@ def get_guidelines_txt1() -> str: class Test_end_to_end_rules1(hunitest.TestCase): - def test_get_header_list1(self) -> None: """ Test extracting headers from a markdown file. @@ -1861,7 +1841,6 @@ def test_extract_rules6(self) -> None: class Test_colorize_bold_text1(hunitest.TestCase): - def test1(self) -> None: """ Test basic case with single bold text. @@ -2024,7 +2003,6 @@ def test8(self) -> None: class Test_format_first_level_bullets1(hunitest.TestCase): - def test1(self) -> None: """ Test basic case with single first level bullet. diff --git a/helpers/test/test_hnumpy.py b/helpers/test/test_hnumpy.py index 81211e439..f536a5a9e 100644 --- a/helpers/test/test_hnumpy.py +++ b/helpers/test/test_hnumpy.py @@ -180,7 +180,7 @@ def test_repr_single1(self) -> None: act = repr(d["test"]) exp = "np.int64(42)" self.assert_equal(act, exp) - + def test_str_full1(self) -> None: """ Test that the __str__ method of OrderedDict returns the expected string. diff --git a/helpers/test/test_hopenai.py b/helpers/test/test_hopenai.py index 8f357c7c3..7f1b529d8 100644 --- a/helpers/test/test_hopenai.py +++ b/helpers/test/test_hopenai.py @@ -4,9 +4,7 @@ import pytest -pytest.importorskip( - "openai" -) # noqa: E402 # pylint: disable=wrong-import-position +pytest.importorskip("openai") # noqa: E402 # pylint: disable=wrong-import-position import helpers.hopenai as hopenai import helpers.hunit_test as hunitest @@ -44,7 +42,11 @@ def _get_openai_request_parameters1() -> dict: messages = hopenai._build_messages( user_prompt=_USER_PROMPT1, system_prompt=_SYSTEM_PROMPT1 ) - data = {"messages": messages, "temperature": _TEMPERATURE1, "model": _MODEL1} + data = { + "messages": messages, + "temperature": _TEMPERATURE1, + "model": _MODEL1, + } return data @@ -62,7 +64,11 @@ def _get_openai_request_parameters2() -> dict: messages = hopenai._build_messages( user_prompt=_USER_PROMPT2, system_prompt=_SYSTEM_PROMPT1 ) - data = {"messages": messages, "temperature": _TEMPERATURE1, "model": _MODEL1} + data = { + "messages": messages, + "temperature": _TEMPERATURE1, + "model": _MODEL1, + } return data @@ -80,7 +86,11 @@ def _get_openai_request_parameters3() -> dict: messages = hopenai._build_messages( user_prompt=_USER_PROMPT1, system_prompt=_SYSTEM_PROMPT1 ) - data = {"messages": messages, "temperature": _TEMPERATURE2, "model": _MODEL1} + data = { + "messages": messages, + "temperature": _TEMPERATURE2, + "model": _MODEL1, + } return data @@ -98,7 +108,11 @@ def _get_openai_request_parameters4() -> dict: messages = hopenai._build_messages( user_prompt=_USER_PROMPT1, system_prompt=_SYSTEM_PROMPT1 ) - data = {"messages": messages, "temperature": _TEMPERATURE2, "model": _MODEL2} + data = { + "messages": messages, + "temperature": _TEMPERATURE2, + "model": _MODEL2, + } return data @@ -122,7 +136,11 @@ def _get_dummy_openai_response1() -> dict: "finish_reason": "stop", } ], - "usage": {"prompt_tokens": 5, "completion_tokens": 4, "total_tokens": 9}, + "usage": { + "prompt_tokens": 5, + "completion_tokens": 4, + "total_tokens": 9, + }, } return response @@ -251,7 +269,6 @@ def tear_down_test(self) -> None: class Test_get_completion(BaseOpenAICacheTest): - def test1(self) -> None: """ Verify that get_completion() returns response from cache with the @@ -278,7 +295,8 @@ def test2(self) -> None: with self.assertRaises(RuntimeError) as RTE: hopenai.get_completion(**parameters4, cache_file=_TEST_CACHE_FILE) self.assert_equal( - str(RTE.exception), "No cached response for this request parameters!" + str(RTE.exception), + "No cached response for this request parameters!", ) @@ -288,7 +306,6 @@ def test2(self) -> None: class Test_hash_key_generator(BaseOpenAICacheTest): - def test_different_request_parameters1(self) -> None: """ This test case check if normalisation works before generating hash key. @@ -326,7 +343,6 @@ def test_different_request_parameters3(self) -> None: class Test_has_cache(BaseOpenAICacheTest): - def test1(self) -> None: """ Should return False if cache doesn't exist. @@ -352,7 +368,6 @@ def test2(self) -> None: class Test_save_response_to_cache(BaseOpenAICacheTest): - def test1(self) -> None: """ Verifies if response saves into cache. @@ -378,7 +393,6 @@ def test1(self) -> None: class Test_load_response_from_cache(BaseOpenAICacheTest): - def test1(self) -> None: """ Verifies if stored response can be loaded. @@ -403,5 +417,7 @@ def test2(self) -> None: parameters4 = _get_openai_request_parameters4() hash_key4 = self.get_completion_cache.hash_key_generator(**parameters4) with self.assertRaises(ValueError) as VE: - self.get_completion_cache.load_response_from_cache(hash_key=hash_key4) + self.get_completion_cache.load_response_from_cache( + hash_key=hash_key4 + ) self.assert_equal(str(VE.exception), "No cache found!") diff --git a/helpers/test/test_hpandas.py b/helpers/test/test_hpandas.py index dc6104e71..04396d94a 100644 --- a/helpers/test/test_hpandas.py +++ b/helpers/test/test_hpandas.py @@ -28,7 +28,6 @@ class Test_dassert_is_unique1(hunitest.TestCase): - def get_df1(self) -> pd.DataFrame: """ Return a df without duplicated index. @@ -107,7 +106,6 @@ def test_dassert_is_unique2(self) -> None: class Test_to_series1(hunitest.TestCase): - def helper(self, n: int, exp: str) -> None: vals = list(range(n)) df = pd.DataFrame([vals], columns=[f"a{i}" for i in vals]) @@ -153,7 +151,6 @@ def test3(self) -> None: class Test_dassert_valid_remap(hunitest.TestCase): - def test1(self) -> None: """ Check that the function works with correct inputs. @@ -263,7 +260,6 @@ def test5(self) -> None: class Test_trim_df1(hunitest.TestCase): - def get_df(self, *args: Any, **kwargs: Any) -> pd.DataFrame: """ Return a df where the CSV txt is read verbatim without inferring dates. @@ -915,9 +911,9 @@ def test_between_idx(self) -> None: ) # Run. start_time = time.time() - filter_values = pd.Series(df.index.get_level_values(ts_col_name)).between( - start_ts, end_ts, inclusive="both" - ) + filter_values = pd.Series( + df.index.get_level_values(ts_col_name) + ).between(start_ts, end_ts, inclusive="both") df = df.droplevel(ts_col_name) df = df[filter_values] end_time = time.time() @@ -1124,7 +1120,6 @@ def test_searchsorted_sorted_idx(self) -> None: class Test_df_to_str(hunitest.TestCase): - @staticmethod def get_test_data() -> pd.DataFrame: test_data = { @@ -1494,7 +1489,10 @@ def test1(self) -> None: "col1": [0.1, 0.2], "col2": ["a", "b c"], "col3": [None, None], - "col4": [pd.Timestamp("2020-01-01"), pd.Timestamp("2021-05-05")], + "col4": [ + pd.Timestamp("2020-01-01"), + pd.Timestamp("2021-05-05"), + ], }, index=[0, 1], ) @@ -1550,7 +1548,6 @@ def test2(self) -> None: class TestDataframeToJson(hunitest.TestCase): - def test_dataframe_to_json(self) -> None: """ Verify correctness of dataframe to JSON transformation. @@ -1595,7 +1592,10 @@ def test_dataframe_to_json_timestamp(self) -> None: # Initialize a dataframe. test_dataframe = pd.DataFrame( { - "col_1": [pd.Timestamp("2020-01-01"), pd.Timestamp("2020-05-12")], + "col_1": [ + pd.Timestamp("2020-01-01"), + pd.Timestamp("2020-05-12"), + ], "col_2": [1.0, 2.0], } ) @@ -1635,7 +1635,6 @@ def test_dataframe_to_json_datetime(self) -> None: class TestFindGapsInDataframes(hunitest.TestCase): - def test_find_gaps_in_dataframes(self) -> None: """ Verify that gaps are caught. @@ -1670,7 +1669,6 @@ def test_find_gaps_in_dataframes(self) -> None: class TestCompareDataframeRows(hunitest.TestCase): - def get_test_data(self) -> pd.DataFrame: test_data = { "dummy_value_1": [0, 1, 3, 2, 0], @@ -1739,7 +1737,6 @@ def test_compare_dataframe_rows2(self) -> None: @pytest.mark.requires_ck_infra @pytest.mark.requires_aws class TestReadDataFromS3(hunitest.TestCase): - def test_read_csv1(self) -> None: s3fs = hs3.get_s3fs(_AWS_PROFILE) file_name = os.path.join( @@ -1771,7 +1768,6 @@ def test_read_parquet1(self) -> None: class TestSubsetDf1(hunitest.TestCase): - def test1(self) -> None: # Generate some random data. np.random.seed(42) @@ -1801,7 +1797,6 @@ def test1(self) -> None: class TestDropNa(hunitest.TestCase): - def test_dropna1(self) -> None: """ Test if all types of NaNs are dropped. @@ -1865,7 +1860,6 @@ def test_dropna2(self) -> None: class TestDropAxisWithAllNans(hunitest.TestCase): - def test_drop_rows1(self) -> None: """ Test if row full of nans is dropped. @@ -3206,7 +3200,9 @@ def get_multiindex_dfs() -> pd.DataFrame: pd.Timestamp("2022-01-01 21:05:00+00:00"), ] iterables1 = [["asset1", "asset2"], ["open", "high", "low", "close"]] - index1 = pd.MultiIndex.from_product(iterables1, names=[None, "timestamp"]) + index1 = pd.MultiIndex.from_product( + iterables1, names=[None, "timestamp"] + ) nums1 = np.array( [ [ @@ -3277,7 +3273,9 @@ def get_multiindex_dfs() -> pd.DataFrame: ["asset1", "asset2", "asset3"], ["open", "high", "low", "close", "volume"], ] - index2 = pd.MultiIndex.from_product(iterables2, names=[None, "timestamp"]) + index2 = pd.MultiIndex.from_product( + iterables2, names=[None, "timestamp"] + ) nums2 = [ [ 0.79095104, @@ -3619,7 +3617,6 @@ def test_intersection2(self) -> None: class Test_compare_nans_in_dataframes(hunitest.TestCase): - def test1(self) -> None: """ Check that NaN differences are identified correctly. @@ -3658,7 +3655,6 @@ def test1(self) -> None: class Test_dassert_increasing_index(hunitest.TestCase): - def test1(self) -> None: """ Check that a monotonically increasing index passes the assert. @@ -3729,7 +3725,6 @@ def test3(self) -> None: class Test_dassert_strictly_increasing_index(hunitest.TestCase): - def test1(self) -> None: """ Check that unique and monotonically increasing index passes the assert. @@ -3809,7 +3804,6 @@ def test3(self) -> None: class Test_apply_index_mode(hunitest.TestCase): - @staticmethod def get_test_data() -> Tuple[pd.DataFrame, pd.DataFrame]: """ @@ -4003,7 +3997,6 @@ def test3(self) -> None: class Test_get_df_from_iterator(hunitest.TestCase): - def test1(self) -> None: """ Check that a dataframe is correctly built from an iterator of @@ -4046,7 +4039,6 @@ def test1(self) -> None: class Test_multiindex_df_info1(hunitest.TestCase): - @staticmethod def get_multiindex_df_with_datetime_index() -> pd.DataFrame: datetime_index = [ @@ -4236,7 +4228,6 @@ def test5(self) -> None: class Test_dassert_index_is_datetime(hunitest.TestCase): - @staticmethod def get_multiindex_df( index_is_datetime: bool, @@ -4338,7 +4329,6 @@ def test4(self) -> None: class Test_dassert_approx_eq1(hunitest.TestCase): - def test1(self) -> None: hpandas.dassert_approx_eq(1, 1.0000001) @@ -4357,7 +4347,6 @@ def test2(self) -> None: class Test_CheckSummary(hunitest.TestCase): - def test1(self) -> None: """ All the tests have passed. @@ -4420,7 +4409,6 @@ def test2(self) -> None: class Test_compute_weighted_sum(hunitest.TestCase): - def helper( self, index1: List[int], @@ -4513,7 +4501,9 @@ def test_compute_weighted_sum4(self) -> None: index_mode = "assert_equal" with self.assertRaises(AssertionError) as cm: hpandas.compute_weighted_sum( - dfs=dfs, weights=pd.DataFrame(weights_data), index_mode=index_mode + dfs=dfs, + weights=pd.DataFrame(weights_data), + index_mode=index_mode, ) actual_signature = str(cm.exception) expected_signature = r""" @@ -4533,7 +4523,6 @@ def test_compute_weighted_sum4(self) -> None: class Test_list_to_str(hunitest.TestCase): - def test1(self) -> None: """ Check that a list is converted to string correctly. @@ -4543,7 +4532,7 @@ def test1(self) -> None: # Run. actual = hpandas.list_to_str(input, enclose_str_char="|", sep_char=" ; ") # Check. - expected = f"5 [|1| ; |two| ; |3| ; |4| ; |five|]" + expected = "5 [|1| ; |two| ; |3| ; |4| ; |five|]" self.assert_equal(actual, expected) def test2(self) -> None: @@ -4555,7 +4544,7 @@ def test2(self) -> None: # Run. actual = hpandas.list_to_str(input, enclose_str_char="", sep_char=" - ") # Check. - expected = f"15 [0 - 1 - 2 - 3 - 4 - ... - 10 - 11 - 12 - 13 - 14]" + expected = "15 [0 - 1 - 2 - 3 - 4 - ... - 10 - 11 - 12 - 13 - 14]" self.assert_equal(actual, expected) def test3(self) -> None: @@ -4568,5 +4557,5 @@ def test3(self) -> None: # Run. actual = hpandas.list_to_str(input) # Check. - expected = f"5 ['1', '2', '3', '4', 'five']" + expected = "5 ['1', '2', '3', '4', 'five']" self.assert_equal(actual, expected) diff --git a/helpers/test/test_hparquet.py b/helpers/test/test_hparquet.py index 2f5e86735..c4bdfd35d 100644 --- a/helpers/test/test_hparquet.py +++ b/helpers/test/test_hparquet.py @@ -116,7 +116,6 @@ def _compare_dfs(self: Any, df1: pd.DataFrame, df2: pd.DataFrame) -> str: class TestParquet1(hunitest.TestCase): - def test_get_df1(self) -> None: """ Check the output of `_get_df()`. @@ -199,7 +198,9 @@ def test_write_and_read_two_columns1(self) -> None: # ////////////////////////////////////////////////////////////////////////////// - def read_filtered_parquet(self, file_name: str, filters: Any) -> pd.DataFrame: + def read_filtered_parquet( + self, file_name: str, filters: Any + ) -> pd.DataFrame: filesystem = None dataset = parquet.ParquetDataset( file_name, @@ -667,7 +668,6 @@ def _run_write_and_read_mixed_units_partitioned_dataset( class TestGetParquetFiltersFromTimestampInterval1(hunitest.TestCase): - def test_no_interval(self) -> None: """ No timestamps provided. @@ -691,9 +691,7 @@ def test_by_month_half1(self) -> None: partition_mode, start_ts, end_ts ) actual = str(filters) - expected = ( - r"[[('year', '==', 2020), ('month', '>=', 1)], [('year', '>', 2020)]]" - ) + expected = r"[[('year', '==', 2020), ('month', '>=', 1)], [('year', '>', 2020)]]" self.assert_equal(actual, expected) def test_by_month_half2(self) -> None: @@ -707,9 +705,7 @@ def test_by_month_half2(self) -> None: partition_mode, start_ts, end_ts ) actual = str(filters) - expected = ( - r"[[('year', '==', 2020), ('month', '<=', 1)], [('year', '<', 2020)]]" - ) + expected = r"[[('year', '==', 2020), ('month', '<=', 1)], [('year', '<', 2020)]]" self.assert_equal(actual, expected) def test_by_month_one_year1(self) -> None: @@ -877,7 +873,6 @@ def test_additional_filters2(self) -> None: class TestAddDatePartitionColumns(hunitest.TestCase): - def add_date_partition_columns_helper( self, partition_mode: str, expected: str ) -> None: @@ -939,7 +934,6 @@ def test_add_date_partition_columns4(self) -> None: class TestToPartitionedDataset(hunitest.TestCase): - @staticmethod def get_test_data1() -> pd.DataFrame: test_data = { @@ -1038,7 +1032,6 @@ def test_to_partitioned_dataset_wrong_column(self) -> None: reason="Run only if CK S3 is available", ) class TestListAndMergePqFiles(hmoto.S3Mock_TestCase): - def generate_test_data(self) -> hs3.AwsProfile: """ Upload test daily Parquet files for 3 days to the mocked S3 bucket. @@ -1077,7 +1070,11 @@ def test_list_and_merge_pq_files(self) -> None: use_relative_paths = True # Check bucket content before merge. parquet_path_list_before = hs3.listdir( - s3_bucket, pattern, only_files, use_relative_paths, aws_profile=s3fs_ + s3_bucket, + pattern, + only_files, + use_relative_paths, + aws_profile=s3fs_, ) self.assertEqual(len(parquet_path_list_before), 6) # Add extra parquet files and rename existing one. @@ -1097,7 +1094,11 @@ def test_list_and_merge_pq_files(self) -> None: s3fs_.copy(renamed_path, additional_path) # Check if edits are in place. updated_parquet_path_list = hs3.listdir( - s3_bucket, pattern, only_files, use_relative_paths, aws_profile=s3fs_ + s3_bucket, + pattern, + only_files, + use_relative_paths, + aws_profile=s3fs_, ) data_parquet_path_list = [ path for path in updated_parquet_path_list if "dummy" not in path @@ -1107,7 +1108,11 @@ def test_list_and_merge_pq_files(self) -> None: # Check bucket content after merge. hparque.list_and_merge_pq_files(self.bucket_name, aws_profile=s3fs_) parquet_path_list_after = hs3.listdir( - s3_bucket, pattern, only_files, use_relative_paths, aws_profile=s3fs_ + s3_bucket, + pattern, + only_files, + use_relative_paths, + aws_profile=s3fs_, ) parquet_path_list_after.sort() expected_list = [ @@ -1157,7 +1162,6 @@ def test_list_and_merge_pq_files_duplicate_drop(self) -> None: class TestListAndMergePqFilesMixedUnits(hunitest.TestCase): - def test_parquet_files_with_mixed_time_units_1(self) -> None: """ Test merging Parquet files with the `ns` and `us`. @@ -1229,7 +1233,6 @@ def _list_and_merge_mixed_units_pq_files( class TestYieldParquetTiles(hunitest.TestCase): - def generate_test_data(self) -> None: """ Generate test data and write it to a scratch dir. @@ -1340,7 +1343,6 @@ def test_yield_tiles_by_year(self) -> None: class TestBuildFilterWithOnlyEqualities(hunitest.TestCase): - def test_year_month_day_equality(self) -> None: """ Test interval with same year, month and day. diff --git a/helpers/test/test_hplayback.py b/helpers/test/test_hplayback.py index 531c3be60..79c1c509a 100644 --- a/helpers/test/test_hplayback.py +++ b/helpers/test/test_hplayback.py @@ -421,7 +421,9 @@ def test1(self) -> None: """ Test writing to file when number of tests is more than generated (10). """ - test_file = hplayba.Playback._get_test_file_name("./path/to/somewhere.py") + test_file = hplayba.Playback._get_test_file_name( + "./path/to/somewhere.py" + ) self.assert_equal( test_file, "./path/to/test/test_by_playback_somewhere.py" ) diff --git a/helpers/test/test_hprint.py b/helpers/test/test_hprint.py index 5d7337c5e..d2feb7a67 100644 --- a/helpers/test/test_hprint.py +++ b/helpers/test/test_hprint.py @@ -14,7 +14,6 @@ class Test_printing1(hunitest.TestCase): - def test_color_highlight1(self) -> None: for c in hprint._COLOR_MAP: _LOG.debug(hprint.color_highlight(c, c)) @@ -26,7 +25,6 @@ def test_color_highlight1(self) -> None: class Test_to_str1(hunitest.TestCase): - def test1(self) -> None: x = 1 # To disable linter complaints. @@ -125,7 +123,6 @@ def example_func5(x: int, y: str, z: float) -> str: class Test_func_signature_to_str1(hunitest.TestCase): - def test1(self) -> None: act = example_func1(1, "hello") exp = "# example_func1: x=1, y='hello'" @@ -158,7 +155,6 @@ def test5(self) -> None: class Test_log(hunitest.TestCase): - def test2(self) -> None: x = 1 # To disable linter complaints. @@ -200,7 +196,6 @@ def test4(self) -> None: class Test_sort_dictionary(hunitest.TestCase): - def test1(self) -> None: dict_ = { "tool": { @@ -257,7 +252,6 @@ def test1(self) -> None: class Test_indent1(hunitest.TestCase): - def test1(self) -> None: txt = """foo @@ -280,7 +274,6 @@ def test1(self) -> None: class Test_dedent1(hunitest.TestCase): - def test1(self) -> None: txt = """ foo @@ -338,7 +331,6 @@ class TestHelloWorld(hunitest.TestCase): class Test_align_on_left1(hunitest.TestCase): - def test1(self) -> None: txt = """foo @@ -360,7 +352,6 @@ def test1(self) -> None: class Test_logging1(hunitest.TestCase): - def test_log_frame1(self) -> None: hprint.log_frame(_LOG, "%s %s", "hello", "world") @@ -377,7 +368,6 @@ def test_log_frame3(self) -> None: class Test_remove_lead_trail_empty_lines1(hunitest.TestCase): - def helper(self, input_str: str, expected_output: List[str]) -> None: """ Test the `remove_lead_trail_empty_lines` function. @@ -408,7 +398,9 @@ def test_single_line_string_returns_single_line_list(self) -> None: expected_output = ["line"] self.helper(input_str, expected_output) - def test_multiple_lines_with_no_empty_lines_returns_same_lines(self) -> None: + def test_multiple_lines_with_no_empty_lines_returns_same_lines( + self, + ) -> None: input_str: str = "line1\nline2\nline3" expected_output = ["line1", "line2", "line3"] self.helper(input_str, expected_output) @@ -450,7 +442,9 @@ def test_single_empty_line_returns_empty_list(self) -> None: expected_output: List[str] = [] self.helper(input_str, expected_output) - def test_multiple_consecutive_empty_lines_at_beginning_and_end(self) -> None: + def test_multiple_consecutive_empty_lines_at_beginning_and_end( + self, + ) -> None: input_str: str = "\n\n\nline1\nline2\n\n\n" expected_output = ["line1", "line2"] self.helper(input_str, expected_output) diff --git a/helpers/test/test_hs3.py b/helpers/test/test_hs3.py index 28fca4f92..e976c80f2 100644 --- a/helpers/test/test_hs3.py +++ b/helpers/test/test_hs3.py @@ -19,7 +19,6 @@ class TestReplaceStarWithDoubleStar(hunitest.TestCase): - def test1(self) -> None: """ Test non replacement of a single asterisk at the end of the path. @@ -74,7 +73,6 @@ def test5(self) -> None: reason="Run only if CK S3 is available", ) class TestToFileAndFromFile1(hmoto.S3Mock_TestCase): - def write_read_helper(self, file_name: str, force_flush: bool) -> None: # Prepare inputs. file_content = "line_mock1\nline_mock2\nline_mock3" @@ -83,7 +81,10 @@ def write_read_helper(self, file_name: str, force_flush: bool) -> None: # Save file. # TODO(Nikola): Is it possible to verify `force_flush`? hs3.to_file( - file_content, s3_path, aws_profile=moto_s3fs, force_flush=force_flush + file_content, + s3_path, + aws_profile=moto_s3fs, + force_flush=force_flush, ) # Read file. saved_content = hs3.from_file(s3_path, aws_profile=moto_s3fs) @@ -161,7 +162,6 @@ def test_from_file_invalid1(self) -> None: reason="Run only if CK S3 is available", ) class TestListdir1(hmoto.S3Mock_TestCase): - def prepare_test_data(self) -> Tuple[str, hs3.AwsProfile]: bucket_s3_path = f"s3://{self.bucket_name}" depth_one_s3_path = f"{bucket_s3_path}/depth_one" @@ -307,7 +307,6 @@ def test_listdir4(self) -> None: reason="Run only if CK S3 is available", ) class TestDu1(hmoto.S3Mock_TestCase): - def test_du1(self) -> None: """ Verify that total file size is returned. @@ -432,7 +431,6 @@ def test2(self) -> None: class Test_get_s3_bucket_from_stage(hunitest.TestCase): - def test1(self) -> None: """ Check for a valid stage. @@ -484,7 +482,6 @@ def test3(self) -> None: @pytest.mark.requires_aws @pytest.mark.requires_ck_infra class Test_s3_get_credentials1(hunitest.TestCase): - def test1(self) -> None: res = hs3.get_aws_credentials(_AWS_PROFILE) _LOG.debug("res=%s", str(res)) @@ -496,7 +493,6 @@ def test1(self) -> None: class Test_s3_functions1(hunitest.TestCase): - def test_extract_bucket_from_path1(self) -> None: path = os.path.join( hs3.get_s3_bucket_path_unit_test(_AWS_PROFILE), @@ -515,7 +511,6 @@ def test_extract_bucket_from_path1(self) -> None: @pytest.mark.requires_aws @pytest.mark.requires_ck_infra class Test_s3_1(hunitest.TestCase): - def test_ls1(self) -> None: file_path = os.path.join( hs3.get_s3_bucket_path_unit_test(_AWS_PROFILE), diff --git a/helpers/test/test_hsecrets.py b/helpers/test/test_hsecrets.py index 7b525aa28..cc046ddac 100644 --- a/helpers/test/test_hsecrets.py +++ b/helpers/test/test_hsecrets.py @@ -8,8 +8,7 @@ import helpers.hgit as hgit assert not hgit.is_cmamp(), ( - "`cmamp` should have moto, while other repos " - "are allowed to not have it)" + "`cmamp` should have moto, while other repos are allowed to not have it)" ) _HAS_MOTO = False @@ -40,7 +39,6 @@ reason="Run only if CK S3 is available", ) class TestCreateClient(hunitest.TestCase): - def test_create_client1(self) -> None: """ Simple smoke test to verify connection to AWS. @@ -55,7 +53,6 @@ def test_create_client1(self) -> None: reason="Run only if CK S3 is available", ) class TestGetSecret(hunitest.TestCase): - @moto.mock_aws def test_get_secret(self) -> None: """ @@ -133,7 +130,6 @@ def test_lock_for_different_script(self) -> None: reason="Run only if CK S3 is available", ) class TestStoreSecret(hunitest.TestCase): - @moto.mock_aws def test_store_secret1(self) -> None: """ @@ -157,9 +153,10 @@ def test_store_secret1(self) -> None: not hserver.is_CK_S3_available(), reason="Run only if CK S3 is available", ) - @pytest.mark.skip(reason="TODO(Juraj): Temporarily disabled in #Cmtask10068.") + @pytest.mark.skip( + reason="TODO(Juraj): Temporarily disabled in #Cmtask10068." + ) class TestLockSecret(hunitest.TestCase): - @moto.mock_aws def test_lock_secret(self) -> None: """ @@ -187,9 +184,10 @@ def test_lock_secret(self) -> None: not hserver.is_CK_S3_available(), reason="Run only if CK S3 is available", ) - @pytest.mark.skip(reason="TODO(Juraj): Temporarily disabled in #Cmtask10068.") + @pytest.mark.skip( + reason="TODO(Juraj): Temporarily disabled in #Cmtask10068." + ) class TestUpdateUsedby(hunitest.TestCase): - @moto.mock_aws def test1(self) -> None: """ diff --git a/helpers/test/test_hserver.py b/helpers/test/test_hserver.py index 722649504..2e8f32556 100644 --- a/helpers/test/test_hserver.py +++ b/helpers/test/test_hserver.py @@ -15,7 +15,6 @@ class _TestCase1: - # def test_config_func_to_str1(self) -> None: # val = hserver.config_func_to_str() # _LOG.info("val=\n%s", val) diff --git a/helpers/test/test_hstring.py b/helpers/test/test_hstring.py index 1a31b465f..49c6e080d 100644 --- a/helpers/test/test_hstring.py +++ b/helpers/test/test_hstring.py @@ -12,7 +12,6 @@ class TestExtractVersionFromFileName(hunitest.TestCase): - def test_extract_version_from_file_name1(self) -> None: """ Verify function provides expected output on valid input. @@ -220,7 +219,6 @@ def test_check_string1(self) -> None: class TestGetCodeBlockLineIndices(hunitest.TestCase): - def helper(self, code: str, expected: List[str]) -> None: lines = code.split("\n") actual_idxs = hstring.get_code_block_line_indices(lines) @@ -251,7 +249,6 @@ def test_assert_equal1(self) -> None: class TestGetDocstrings(hunitest.TestCase): - def test1(self) -> None: """ Test that grouped lines within docstrings are correctly returned. diff --git a/helpers/test/test_hunit_test.py b/helpers/test/test_hunit_test.py index a0e719231..b207d4f32 100644 --- a/helpers/test/test_hunit_test.py +++ b/helpers/test/test_hunit_test.py @@ -145,7 +145,9 @@ def test_get_scratch_space2(self) -> None: test_class_name=test_class_name, test_method_name=test_method_name ) act = hunitest.purify_txt_from_client(act) - exp = "$GIT_ROOT/helpers/test/outcomes/test_class.test_method/tmp.scratch" + exp = ( + "$GIT_ROOT/helpers/test/outcomes/test_class.test_method/tmp.scratch" + ) self.assertEqual(act, exp) def test_get_scratch_space3(self) -> None: @@ -192,7 +194,9 @@ def test_assert_not_equal2(self) -> None: expected = "hello world w" # Create a dir like `/var/tmp/tmph_kun9xq`. tmp_dir = tempfile.mkdtemp() - self.assert_equal(actual, expected, abort_on_error=False, dst_dir=tmp_dir) + self.assert_equal( + actual, expected, abort_on_error=False, dst_dir=tmp_dir + ) # Compute the signature from the dir. act = hunitest.get_dir_signature( tmp_dir, include_file_content=True, num_lines=None @@ -263,7 +267,6 @@ def _remove_lines1(self) -> None: class Test_AssertEqual1(hunitest.TestCase): - def test_equal1(self) -> None: """ Matching act and exp without fuzzy matching. @@ -387,7 +390,6 @@ def test_not_equal_debug(self) -> None: class TestCheckString1(hunitest.TestCase): - def test_check_string1(self) -> None: """ Compare the actual value to a matching golden outcome. @@ -879,7 +881,6 @@ def _check_df_helper( class Test_check_string_debug1(hunitest.TestCase): - def test1(self) -> None: act = "hello" # action_on_missing_golden = "assert" @@ -901,7 +902,6 @@ def test2(self) -> None: class Test_unit_test1(hunitest.TestCase): - def test_purify_txt_from_client1(self) -> None: super_module_path = hgit.get_client_root(super_module=True) # TODO(gp): We should remove the current path. @@ -950,7 +950,6 @@ def test_purify_txt_from_client2(self) -> None: class Test_unit_test2(hunitest.TestCase): - def test_purify_parquet_file_names1(self) -> None: """ Test purification of Parquet file names with the path. @@ -989,7 +988,6 @@ def test_purify_parquet_file_names2(self) -> None: class Test_get_dir_signature1(hunitest.TestCase): - def helper(self, include_file_content: bool) -> str: in_dir = self.get_input_dir() act = hunitest.get_dir_signature( @@ -1036,7 +1034,6 @@ def test2(self) -> None: class Test_purify_txt_from_client1(hunitest.TestCase): - def helper(self, txt: str, exp: str) -> None: act = hunitest.purify_txt_from_client(txt) self.assert_equal(act, exp) @@ -1115,7 +1112,6 @@ def test1(self) -> None: class Test_purify_object_representation1(hunitest.TestCase): - def helper(self, txt: str, exp: str) -> None: txt = hprint.dedent(txt) act = hunitest.purify_object_representation(txt) @@ -1220,7 +1216,6 @@ def test4(self) -> None: class Test_purify_amp_reference1(hunitest.TestCase): - def helper(self, txt: str, exp: str) -> None: txt = hprint.dedent(txt) act = hunitest.purify_amp_references(txt) @@ -1250,7 +1245,6 @@ def test1(self) -> None: class Test_purify_from_environment1(hunitest.TestCase): - def check_helper(self, input_: str, exp: str) -> None: """ Check that the text is purified from environment variables correctly. @@ -1316,7 +1310,6 @@ def test8(self) -> None: class Test_purify_line_number1(hunitest.TestCase): - def test1(self) -> None: """ Check that the text is purified from line numbers correctly. @@ -1341,7 +1334,6 @@ def test1(self) -> None: class Test_purify_docker_image_name1(hunitest.TestCase): - def test1(self) -> None: txt = r""" docker run --rm --user $(id -u):$(id -g) --workdir $GIT_ROOT --mount type=bind,source=/Users/saggese/src/helpers1,target=$GIT_ROOT tmp.latex.edb567be pdflatex -output-directory diff --git a/helpers/test/test_hunit_test_mock.py b/helpers/test/test_hunit_test_mock.py index fd433a044..c29628918 100644 --- a/helpers/test/test_hunit_test_mock.py +++ b/helpers/test/test_hunit_test_mock.py @@ -24,7 +24,6 @@ def _check(self: Any, str_to_eval: str, exp_val: str) -> None: class _Class: - def __init__(self) -> None: self.a = 3 self.b = 14 @@ -42,7 +41,6 @@ def get_b(self) -> int: class _TestCase(hunitest.TestCase): - def check(self, *args, **kwargs) -> None: _check(self, *args, **kwargs) @@ -190,7 +188,6 @@ def test_str1(self) -> None: class Test_Mock_Class1(_TestCase): - def test_without_mock1(self) -> None: obj = _Class() self.assertEqual(obj.get_a(), 3) @@ -236,7 +233,6 @@ def test_with_mock2(self) -> None: class Test_Mock_Class_with_decorator1(_TestCase): - @umock.patch.object(_Class, "get_a", return_value=4) def test1(self, mock_method: umock.MagicMock) -> None: """ @@ -245,7 +241,9 @@ def test1(self, mock_method: umock.MagicMock) -> None: obj = _Class() # Check. # self.assertIs(mock_method, umock.MagicMock) - self.check("mock_method", "mock_method=") + self.check( + "mock_method", "mock_method=" + ) self.assertEqual(obj.get_a(), 4) mock_method.assert_called() obj.get_a.assert_called() @@ -257,7 +255,6 @@ def test1(self, mock_method: umock.MagicMock) -> None: class Test_Mock_Class_with_context_manager1(_TestCase): - def test1(self) -> None: """ Patch an object method using a context manager. @@ -266,7 +263,9 @@ def test1(self) -> None: with umock.patch.object(_Class, "get_a", return_value=4): obj = _Class() # Check. - self.check("obj.get_a", "obj.get_a=") + self.check( + "obj.get_a", "obj.get_a=" + ) self.assertEqual(obj.get_a(), 4) obj.get_a.assert_called() # Outside the context manager everything is normal. diff --git a/helpers/test/test_joblib_helpers.py b/helpers/test/test_joblib_helpers.py index 404e5776b..e17d62b98 100644 --- a/helpers/test/test_joblib_helpers.py +++ b/helpers/test/test_joblib_helpers.py @@ -130,7 +130,12 @@ def _run_test(self, num_threads: Union[str, int], backend: str) -> None: # expected_return = self.EXPECTED_RETURN _helper_success( - self, workload, num_threads, abort_on_error, expected_return, backend + self, + workload, + num_threads, + abort_on_error, + expected_return, + backend, ) diff --git a/helpers/test/test_lib_tasks.py b/helpers/test/test_lib_tasks.py index 04ee72d3a..0f5f55a3c 100644 --- a/helpers/test/test_lib_tasks.py +++ b/helpers/test/test_lib_tasks.py @@ -324,12 +324,15 @@ def test_docker_stats(self) -> None: reason="Only run in amp as supermodule", ) def test_gh_create_pr1(self) -> None: - with umock.patch.object( - hgit, "get_branch_name", return_value="AmpTask1_test_branch" - ), umock.patch.object( - hlitagh, - "_get_repo_full_name_from_cmd", - return_value=("github.com/alphamatic/amp", "amp"), + with ( + umock.patch.object( + hgit, "get_branch_name", return_value="AmpTask1_test_branch" + ), + umock.patch.object( + hlitagh, + "_get_repo_full_name_from_cmd", + return_value=("github.com/alphamatic/amp", "amp"), + ), ): target = "gh_create_pr(ctx, title='test')" self._check_output(target) @@ -341,12 +344,15 @@ def test_gh_create_pr1(self) -> None: reason="Only run in amp as supermodule", ) def test_gh_create_pr2(self) -> None: - with umock.patch.object( - hgit, "get_branch_name", return_value="AmpTask1_test_branch" - ), umock.patch.object( - hlitagh, - "_get_repo_full_name_from_cmd", - return_value=("github.com/alphamatic/amp", "amp"), + with ( + umock.patch.object( + hgit, "get_branch_name", return_value="AmpTask1_test_branch" + ), + umock.patch.object( + hlitagh, + "_get_repo_full_name_from_cmd", + return_value=("github.com/alphamatic/amp", "amp"), + ), ): target = "gh_create_pr(ctx, body='hello_world', title='test')" self._check_output(target) @@ -358,12 +364,15 @@ def test_gh_create_pr2(self) -> None: reason="Only run in amp as supermodule", ) def test_gh_create_pr3(self) -> None: - with umock.patch.object( - hgit, "get_branch_name", return_value="AmpTask1_test_branch" - ), umock.patch.object( - hlitagh, - "_get_repo_full_name_from_cmd", - return_value=("github.com/alphamatic/amp", "amp"), + with ( + umock.patch.object( + hgit, "get_branch_name", return_value="AmpTask1_test_branch" + ), + umock.patch.object( + hlitagh, + "_get_repo_full_name_from_cmd", + return_value=("github.com/alphamatic/amp", "amp"), + ), ): target = "gh_create_pr(ctx, draft=False, title='test')" self._check_output(target) diff --git a/helpers/test/test_lib_tasks_docker.py b/helpers/test/test_lib_tasks_docker.py index a5f090950..d0868f662 100644 --- a/helpers/test/test_lib_tasks_docker.py +++ b/helpers/test/test_lib_tasks_docker.py @@ -24,7 +24,6 @@ class Test_generate_compose_file1(hunitest.TestCase): - def helper( self, stage: str, @@ -108,7 +107,6 @@ def test5(self) -> None: class Test_generate_compose_file2(hunitest.TestCase): - def helper( self, mock_getcwd: str, @@ -138,16 +136,19 @@ def helper( txt.append(txt_tmp) # file_name = None - with umock.patch.object( - os, "getcwd", return_value=mock_getcwd - ), umock.patch.object( - hgit, "find_git_root", return_value=mock_find_git_root - ), umock.patch.object( - hgit, "find_helpers_root", return_value=mock_find_helpers_root - ), umock.patch.object( - hgit, - "is_in_helpers_as_supermodule", - return_value=mock_is_in_helpers_as_supermodule, + with ( + umock.patch.object(os, "getcwd", return_value=mock_getcwd), + umock.patch.object( + hgit, "find_git_root", return_value=mock_find_git_root + ), + umock.patch.object( + hgit, "find_helpers_root", return_value=mock_find_helpers_root + ), + umock.patch.object( + hgit, + "is_in_helpers_as_supermodule", + return_value=mock_is_in_helpers_as_supermodule, + ), ): txt_tmp = hlitadoc._generate_docker_compose_file( stage, @@ -422,7 +423,6 @@ def test_docker_jupyter1(self) -> None: class Test_dassert_is_image_name_valid1(hunitest.TestCase): - def test1(self) -> None: """ Check that valid images pass the assertion. @@ -463,7 +463,6 @@ def test2(self) -> None: class Test_dassert_is_base_image_name_valid1(hunitest.TestCase): - def test1(self) -> None: """ Check that valid base images pass the assertion. diff --git a/helpers/test/test_lib_tasks_docker_release.py b/helpers/test/test_lib_tasks_docker_release.py index d76aaade2..ac3f809d8 100644 --- a/helpers/test/test_lib_tasks_docker_release.py +++ b/helpers/test/test_lib_tasks_docker_release.py @@ -454,7 +454,9 @@ def test_dockerhub1(self) -> None: # ############################################################################# -class Test_docker_tag_push_multi_build_local_image_as_dev1(_DockerFlowTestHelper): +class Test_docker_tag_push_multi_build_local_image_as_dev1( + _DockerFlowTestHelper +): """ Test tagging and pushing a multi-arch local Docker image as dev. """ @@ -923,7 +925,9 @@ def set_up_test2(self) -> None: self.mock_workspace_check = self.workspace_check_patcher.start() self.patchers["workspace_check"] = self.workspace_check_patcher # Mock file existence check to handle both paths. - self.file_exists_patcher = umock.patch("helpers.hdbg.dassert_file_exists") + self.file_exists_patcher = umock.patch( + "helpers.hdbg.dassert_file_exists" + ) self.mock_file_exists = self.file_exists_patcher.start() self.patchers["file_exists"] = self.file_exists_patcher # Mock `docker_build_prod_image()`. diff --git a/helpers/test/test_lib_tasks_git.py b/helpers/test/test_lib_tasks_git.py index c9e84662a..1707bb630 100644 --- a/helpers/test/test_lib_tasks_git.py +++ b/helpers/test/test_lib_tasks_git.py @@ -29,10 +29,14 @@ def helper( ctx = httestlib._build_mock_context_returning_ok() # mode = "tar" - hlitagit.git_patch_create(ctx, mode, modified, branch, last_commit, files) + hlitagit.git_patch_create( + ctx, mode, modified, branch, last_commit, files + ) # mode = "diff" - hlitagit.git_patch_create(ctx, mode, modified, branch, last_commit, files) + hlitagit.git_patch_create( + ctx, mode, modified, branch, last_commit, files + ) def test_tar_modified1(self) -> None: """ @@ -94,7 +98,9 @@ def test_tar_files1(self) -> None: branch = False last_commit = False files = __file__ - hlitagit.git_patch_create(ctx, mode, modified, branch, last_commit, files) + hlitagit.git_patch_create( + ctx, mode, modified, branch, last_commit, files + ) def test_diff_files_abort1(self) -> None: """ diff --git a/helpers/test/test_lib_tasks_integrate.py b/helpers/test/test_lib_tasks_integrate.py index 560c76d08..0651f81f6 100644 --- a/helpers/test/test_lib_tasks_integrate.py +++ b/helpers/test/test_lib_tasks_integrate.py @@ -7,10 +7,12 @@ def test1(self) -> None: # Define input variables. src_dir = "/src/cmamp1/oms/broker/broker.py" # Call function to test. - act = hlitaint._infer_dst_file_path(src_dir, + act = hlitaint._infer_dst_file_path( + src_dir, default_src_dir_basename="cmamp1", default_dst_dir_basename="amp1", - check_exists=False) + check_exists=False, + ) # Define expected output. exp = ( "/src/amp1/oms/broker/broker.py", diff --git a/helpers/test/test_lib_tasks_pytest.py b/helpers/test/test_lib_tasks_pytest.py index 8f25d773e..0dd3532c3 100644 --- a/helpers/test/test_lib_tasks_pytest.py +++ b/helpers/test/test_lib_tasks_pytest.py @@ -27,7 +27,6 @@ class Test_build_run_command_line1(hunitest.TestCase): - def run_fast_tests1_helper( self, is_dev_csfy_return_value: bool, @@ -51,10 +50,13 @@ def run_fast_tests1_helper( tee_to_file = False n_threads = "1" # - with umock.patch.object( - hserver, "is_dev_csfy", return_value=is_dev_csfy_return_value - ), umock.patch.object( - hserver, "is_inside_ci", return_value=is_inside_ci_return_value + with ( + umock.patch.object( + hserver, "is_dev_csfy", return_value=is_dev_csfy_return_value + ), + umock.patch.object( + hserver, "is_inside_ci", return_value=is_inside_ci_return_value + ), ): act = hlitapyt._build_run_command_line( "fast_tests", @@ -132,10 +134,13 @@ def run_fast_tests2_helper( tee_to_file = False n_threads = "1" # - with umock.patch.object( - hserver, "is_dev_csfy", return_value=is_dev_csfy_return_value - ), umock.patch.object( - hserver, "is_inside_ci", return_value=is_inside_ci_return_value + with ( + umock.patch.object( + hserver, "is_dev_csfy", return_value=is_dev_csfy_return_value + ), + umock.patch.object( + hserver, "is_inside_ci", return_value=is_inside_ci_return_value + ), ): act = hlitapyt._build_run_command_line( "fast_tests", @@ -275,10 +280,13 @@ def run_fast_tests5_helper( tee_to_file = True n_threads = "1" # - with umock.patch.object( - hserver, "is_dev_csfy", return_value=is_dev_csfy_return_value - ), umock.patch.object( - hserver, "is_inside_ci", return_value=is_inside_ci_return_value + with ( + umock.patch.object( + hserver, "is_dev_csfy", return_value=is_dev_csfy_return_value + ), + umock.patch.object( + hserver, "is_inside_ci", return_value=is_inside_ci_return_value + ), ): act = hlitapyt._build_run_command_line( "fast_tests", @@ -358,10 +366,13 @@ def run_fast_tests6_helper( tee_to_file = False n_threads = "1" # - with umock.patch.object( - hserver, "is_dev_csfy", return_value=is_dev_csfy_return_value - ), umock.patch.object( - hserver, "is_inside_ci", return_value=is_inside_ci_return_value + with ( + umock.patch.object( + hserver, "is_dev_csfy", return_value=is_dev_csfy_return_value + ), + umock.patch.object( + hserver, "is_inside_ci", return_value=is_inside_ci_return_value + ), ): act = hlitapyt._build_run_command_line( "fast_tests", @@ -439,10 +450,13 @@ def run_fast_tests7_helper( tee_to_file = False n_threads = "auto" # - with umock.patch.object( - hserver, "is_dev_csfy", return_value=is_dev_csfy_return_value - ), umock.patch.object( - hserver, "is_inside_ci", return_value=is_inside_ci_return_value + with ( + umock.patch.object( + hserver, "is_dev_csfy", return_value=is_dev_csfy_return_value + ), + umock.patch.object( + hserver, "is_inside_ci", return_value=is_inside_ci_return_value + ), ): act = hlitapyt._build_run_command_line( "fast_tests", @@ -528,10 +542,13 @@ def get_custom_marker_helper( tee_to_file = False n_threads = "1" # Mock test. - with umock.patch.object( - hserver, "is_dev_csfy", return_value=is_dev_csfy_return_value - ), umock.patch.object( - hserver, "is_inside_ci", return_value=is_inside_ci_return_value + with ( + umock.patch.object( + hserver, "is_dev_csfy", return_value=is_dev_csfy_return_value + ), + umock.patch.object( + hserver, "is_inside_ci", return_value=is_inside_ci_return_value + ), ): custom_marker = hlitapyt._get_custom_marker( run_only_test_list=run_only_test_list, @@ -602,7 +619,6 @@ def get_custom_marker2_empty(self) -> None: class Test_pytest_repro1(hunitest.TestCase): - def helper(self, file_name: str, mode: str, exp: List[str]) -> None: script_name = os.path.join( self.get_scratch_space(), "tmp.pytest_repro.sh" @@ -969,7 +985,6 @@ def test7(self) -> None: class Test_pytest_failed1(hunitest.TestCase): - def get_pytest_text1(self) -> str: txt = """ 20:48:15 - ^[[36mINFO ^[[0m hdbg.py init_logger:1018 > cmd='/venv/bin/pytest helpers_root/dev_scripts_helpers/documentation/' diff --git a/helpers/test/test_master_buildmeister_dashboard.py b/helpers/test/test_master_buildmeister_dashboard.py index 773f7f205..ac46b6c17 100644 --- a/helpers/test/test_master_buildmeister_dashboard.py +++ b/helpers/test/test_master_buildmeister_dashboard.py @@ -27,7 +27,6 @@ def build_config() -> cconfig.ConfigList: class Test_Master_buildmeister_dashboard_notebook( dshnrntca.Test_Run_Notebook_TestCase ): - @pytest.mark.skipif( not hserver.is_inside_ci(), reason="No access to data from `lemonade` repo locally", @@ -40,7 +39,10 @@ class Test_Master_buildmeister_dashboard_notebook( def test1(self) -> None: amp_dir = hgit.get_amp_abs_path() notebook_path = os.path.join( - amp_dir, "devops", "notebooks", "Master_buildmeister_dashboard.ipynb" + amp_dir, + "devops", + "notebooks", + "Master_buildmeister_dashboard.ipynb", ) config_builder = ( "helpers.test.test_master_buildmeister_dashboard.build_config()" diff --git a/helpers/test/test_repo_config_amp.py b/helpers/test/test_repo_config_amp.py index f9960a7b5..82ce68847 100644 --- a/helpers/test/test_repo_config_amp.py +++ b/helpers/test/test_repo_config_amp.py @@ -86,7 +86,6 @@ def test_is_CK_S3_available(self) -> None: class TestRepoConfig_Amp_signature1(hunitest.TestCase): - def test_dev_csfy_server(self) -> None: target_name = "amp" hunteuti.execute_only_in_target_repo(target_name) diff --git a/helpers/test/test_repo_config_utils.py b/helpers/test/test_repo_config_utils.py index 93c7850be..b72fdc26a 100644 --- a/helpers/test/test_repo_config_utils.py +++ b/helpers/test/test_repo_config_utils.py @@ -15,7 +15,6 @@ class Test_repo_config1(hunitest.TestCase): - def create_test_file(self) -> str: yaml_txt = """ repo_info: diff --git a/helpers/unit_test_template.py b/helpers/unit_test_template.py index 57d11f8b8..1b769e1b4 100644 --- a/helpers/unit_test_template.py +++ b/helpers/unit_test_template.py @@ -17,6 +17,5 @@ class Test_Example(hunitest.TestCase): - def test_example1(self) -> None: pass diff --git a/import_check/detect_import_cycles.py b/import_check/detect_import_cycles.py index cf71a3873..7dacf70be 100755 --- a/import_check/detect_import_cycles.py +++ b/import_check/detect_import_cycles.py @@ -59,7 +59,8 @@ def _check_import_cycles( def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("module", type=str, help="Path to the target module") hparser.add_verbosity_arg(parser) diff --git a/import_check/show_imports.py b/import_check/show_imports.py index 42131564a..8c2dfdfab 100755 --- a/import_check/show_imports.py +++ b/import_check/show_imports.py @@ -617,7 +617,7 @@ def _remove_isolated_nodes_from_graph(self) -> None: # Store the nodes that are connected to at least one other node. connected_nodes = set() new_structured_graph = copy.deepcopy(self.structured_graph) - for (n1, n2) in self.structured_graph.edges: + for n1, n2 in self.structured_graph.edges: connected_nodes.add(n1) connected_nodes.add(n2) # Remove all the other nodes. @@ -901,7 +901,8 @@ def _show_dependencies( def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("module", type=str, help="Path to the target module") parser.add_argument( diff --git a/import_check/test/test_detect_import_cycles.py b/import_check/test/test_detect_import_cycles.py index 59fc32b2a..48ceb153a 100644 --- a/import_check/test/test_detect_import_cycles.py +++ b/import_check/test/test_detect_import_cycles.py @@ -163,12 +163,12 @@ def test6(self) -> None: files = {} for dir_ in dirs: files[f"{dir_}/__init__.py"] = "" - files[ - "subdir1/file1.py" - ] = f"import {in_dir_name}.subdir2.subdir3.file1\n" - files[ - "subdir2/subdir3/file1.py" - ] = f"import {in_dir_name}.subdir1.file1\n" + files["subdir1/file1.py"] = ( + f"import {in_dir_name}.subdir2.subdir3.file1\n" + ) + files["subdir2/subdir3/file1.py"] = ( + f"import {in_dir_name}.subdir1.file1\n" + ) files["__init__.py"] = "" # Run and check the outcome. expected = [ @@ -195,12 +195,12 @@ def test7(self) -> None: files = {} for dir_ in dirs: files[f"{dir_}/__init__.py"] = "" - files[ - "subdir1/subdir2/subdir3/file1.py" - ] = f"import {in_dir_name}.subdir4.subdir5.file1\n" - files[ - "subdir4/subdir5/file1.py" - ] = f"import {in_dir_name}.subdir1.subdir2.subdir3.file1\n" + files["subdir1/subdir2/subdir3/file1.py"] = ( + f"import {in_dir_name}.subdir4.subdir5.file1\n" + ) + files["subdir4/subdir5/file1.py"] = ( + f"import {in_dir_name}.subdir1.subdir2.subdir3.file1\n" + ) files["__init__.py"] = "" # Run and check the outcome. expected = [ @@ -306,12 +306,12 @@ def test11(self) -> None: in_dir_name = self.get_input_dir().split("/")[-1] dirs = ["subdir1", "subdir1/subdir2"] files = {} - files[ - "subdir1/subdir2/file1.py" - ] = f"import {in_dir_name}.subdir1.subdir2.file2\n" - files[ - "subdir1/subdir2/file2.py" - ] = f"import {in_dir_name}.subdir1.subdir2.file1\n" + files["subdir1/subdir2/file1.py"] = ( + f"import {in_dir_name}.subdir1.subdir2.file2\n" + ) + files["subdir1/subdir2/file2.py"] = ( + f"import {in_dir_name}.subdir1.subdir2.file1\n" + ) files["subdir1/subdir2/__init__.py"] = "" files["__init__.py"] = "" # Run and check the outcome. @@ -366,12 +366,12 @@ def test13(self) -> None: in_dir_name = self.get_input_dir().split("/")[-1] dirs = ["subdir1", "subdir1/subdir2"] files = {} - files[ - "subdir1/subdir2/file1.py" - ] = f"import {in_dir_name}.subdir1.subdir2.file2\n" - files[ - "subdir1/subdir2/file2.py" - ] = f"import {in_dir_name}.subdir1.subdir2.file1\n" + files["subdir1/subdir2/file1.py"] = ( + f"import {in_dir_name}.subdir1.subdir2.file2\n" + ) + files["subdir1/subdir2/file2.py"] = ( + f"import {in_dir_name}.subdir1.subdir2.file1\n" + ) files["subdir1/subdir2/__init__.py"] = "" # Run and check the outcome. expected = ( diff --git a/import_check/test/test_show_imports.py b/import_check/test/test_show_imports.py index d5c1f30d6..07c7998a1 100644 --- a/import_check/test/test_show_imports.py +++ b/import_check/test/test_show_imports.py @@ -262,9 +262,9 @@ def test11(self) -> None: in_dir_name = self.get_input_dir().split("/")[-1] files = {} files["file1.py"] = "import numpy\n" - files[ - "file2.py" - ] = f"import {in_dir_name}.file1\nimport {in_dir_name}.file3\n" + files["file2.py"] = ( + f"import {in_dir_name}.file1\nimport {in_dir_name}.file3\n" + ) files["file3.py"] = f"import {in_dir_name}.file2\n" files["__init__.py"] = "" # Run and check the outcome. diff --git a/linters/action.py b/linters/action.py index 915841dad..32e31ceb4 100644 --- a/linters/action.py +++ b/linters/action.py @@ -199,7 +199,6 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: class CompositeAction(Action): - def __init__(self, actions: List[Action]) -> None: super().__init__() self._actions = actions diff --git a/linters/add_python_init_files.py b/linters/add_python_init_files.py index 4ab507ae9..a0ca0950b 100644 --- a/linters/add_python_init_files.py +++ b/linters/add_python_init_files.py @@ -67,7 +67,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_add_class_frames.py b/linters/amp_add_class_frames.py index 506434a33..bf80e4e7e 100644 --- a/linters/amp_add_class_frames.py +++ b/linters/amp_add_class_frames.py @@ -92,7 +92,7 @@ class frames (see below). break if ( j > 1 - and lines[j] == lines[j - 2] == f"# {'#' * (MAX_LINE_LENGTH-2)}" + and lines[j] == lines[j - 2] == f"# {'#' * (MAX_LINE_LENGTH - 2)}" and re.match(r"#\s\w+", lines[j - 1]) ): # There is already a frame above this class that needs to be removed. @@ -137,9 +137,9 @@ def _insert_frame( # Build the class frame. class_name = class_init_match.group(1) class_frame = [ - f"# {'#' * (MAX_LINE_LENGTH-2)}", + f"# {'#' * (MAX_LINE_LENGTH - 2)}", f"# {class_name}", - f"# {'#' * (MAX_LINE_LENGTH-2)}", + f"# {'#' * (MAX_LINE_LENGTH - 2)}", "", ] # Add the class frame to the lines to be written into the updated file. @@ -178,7 +178,6 @@ def update_class_frames(file_content: str) -> List[str]: class _ClassFramer(liaction.Action): - def check_if_possible(self) -> bool: return True @@ -199,7 +198,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_add_toc_to_notebook.py b/linters/amp_add_toc_to_notebook.py index 1d27452cc..b3477350e 100644 --- a/linters/amp_add_toc_to_notebook.py +++ b/linters/amp_add_toc_to_notebook.py @@ -25,7 +25,6 @@ class _AddTOC(liaction.Action): - def __init__(self) -> None: executable = "$(find -wholename '*dev_scripts_helpers/notebooks/add_toc_to_notebook.py')" super().__init__(executable) @@ -61,7 +60,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_autoflake.py b/linters/amp_autoflake.py index 2f2a1523c..4a696ae84 100644 --- a/linters/amp_autoflake.py +++ b/linters/amp_autoflake.py @@ -66,7 +66,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--abort_on_change", diff --git a/linters/amp_black.py b/linters/amp_black.py index ba17f65d0..2053bbe45 100644 --- a/linters/amp_black.py +++ b/linters/amp_black.py @@ -8,6 +8,7 @@ import linters.amp_black as lampblac """ + import argparse import logging import os @@ -59,7 +60,9 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: # - 1 file left unchanged. to_remove = ["All done!", "file left unchanged", "reformatted"] output = [ - line for line in output if all(word not in line for word in to_remove) + line + for line in output + if all(word not in line for word in to_remove) ] return output @@ -69,7 +72,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--abort_on_change", diff --git a/linters/amp_check_file_size.py b/linters/amp_check_file_size.py index fd855940e..de3702860 100644 --- a/linters/amp_check_file_size.py +++ b/linters/amp_check_file_size.py @@ -40,7 +40,6 @@ def _check_file_size(file_name: str, max_kb: int = 500) -> str: class _FileSizeChecker(liaction.Action): - def check_if_possible(self) -> bool: return True @@ -52,7 +51,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_check_filename.py b/linters/amp_check_filename.py index b512989b5..471538143 100644 --- a/linters/amp_check_filename.py +++ b/linters/amp_check_filename.py @@ -8,6 +8,7 @@ import linters.amp_check_filename as lamchfil """ + import argparse import logging import os @@ -113,7 +114,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "-e", diff --git a/linters/amp_check_import.py b/linters/amp_check_import.py index b9b6ee458..2b54e2212 100644 --- a/linters/amp_check_import.py +++ b/linters/amp_check_import.py @@ -8,6 +8,7 @@ import linters.amp_check_import as lamchimp """ + import argparse import logging import re @@ -55,7 +56,6 @@ def _check_import(file_name: str, line_num: int, line: str) -> str: class _CheckImport(liaction.Action): - def check_if_possible(self) -> bool: return True @@ -75,7 +75,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_check_md_reference.py b/linters/amp_check_md_reference.py index 343760b6a..b4fc9c046 100644 --- a/linters/amp_check_md_reference.py +++ b/linters/amp_check_md_reference.py @@ -85,7 +85,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_check_md_toc_headers.py b/linters/amp_check_md_toc_headers.py index 6a48b5fa9..c723be62d 100644 --- a/linters/amp_check_md_toc_headers.py +++ b/linters/amp_check_md_toc_headers.py @@ -101,7 +101,9 @@ def verify_toc_position(lines: List[str], file_name: str) -> List[str]: return warnings -def fix_md_toc_headers(file_name: str) -> Tuple[List[str], List[str], List[str]]: +def fix_md_toc_headers( + file_name: str, +) -> Tuple[List[str], List[str], List[str]]: """ Fix header levels and verify the Table of Contents. @@ -126,7 +128,6 @@ def fix_md_toc_headers(file_name: str) -> Tuple[List[str], List[str], List[str]] class _TOCHeaderFixer(liaction.Action): - def check_if_possible(self) -> bool: return True @@ -147,7 +148,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_check_merge_conflict.py b/linters/amp_check_merge_conflict.py index ab3d7adb3..b17cb8d3f 100644 --- a/linters/amp_check_merge_conflict.py +++ b/linters/amp_check_merge_conflict.py @@ -67,7 +67,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_check_shebang.py b/linters/amp_check_shebang.py index 17fc65b6a..c56e508e7 100644 --- a/linters/amp_check_shebang.py +++ b/linters/amp_check_shebang.py @@ -70,7 +70,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_class_method_order.py b/linters/amp_class_method_order.py index a1a8a7144..097d31ca5 100644 --- a/linters/amp_class_method_order.py +++ b/linters/amp_class_method_order.py @@ -70,10 +70,7 @@ def _sorting_key(node: Union[cst.ClassDef, cst.FunctionDef]) -> int: # ############################################################################# -class _OrderMethods( - codemod.ContextAwareTransformer -): # pylint: disable=too-many-ancestors - +class _OrderMethods(codemod.ContextAwareTransformer): # pylint: disable=too-many-ancestors def leave_ClassDef( self, original_node: cst.ClassDef, updated_node: cst.ClassDef ) -> Union[cst.BaseStatement, cst.RemovalSentinel]: @@ -156,7 +153,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_doc_formatter.py b/linters/amp_doc_formatter.py index 191187b02..1ba8566ec 100644 --- a/linters/amp_doc_formatter.py +++ b/linters/amp_doc_formatter.py @@ -8,6 +8,7 @@ import linters.amp_doc_formatter as lamdofor """ + import argparse import logging import re @@ -87,7 +88,9 @@ def _find_unbalanced_triple_backticks(file_name: str) -> List[int]: # Convert zero-indexed numbers to one-indexed line numbers. # This would accurately link it to the part of the code; # Where the docstring starts on code editors. - idxs_docstrings_with_unbalanced_backticks.append(docstring[0] + 1) + idxs_docstrings_with_unbalanced_backticks.append( + docstring[0] + 1 + ) return idxs_docstrings_with_unbalanced_backticks @staticmethod @@ -121,9 +124,7 @@ def _remove_ignored_docstrings(file_name: str) -> Dict[str, str]: if match is None: break hash_id = str(uuid.uuid4()) - contents = ( - f"{contents[: match.start()]}# {hash_id}{contents[match.end() :]}" - ) + contents = f"{contents[: match.start()]}# {hash_id}{contents[match.end() :]}" result[hash_id] = match.group(0) hio.to_file(file_name, contents) return result @@ -269,7 +270,6 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: class _Pydocstyle(liaction.Action): - def __init__(self) -> None: executable = "pydocstyle" super().__init__(executable) @@ -406,7 +406,6 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: # TODO(gp): Fix this. # Not installable through conda. class _Pyment(liaction.Action): - def __init__(self) -> None: executable = "pyment" super().__init__(executable) @@ -471,7 +470,8 @@ def __init__(self) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_fix_comments.py b/linters/amp_fix_comments.py index 8e2c97501..fddd605ea 100644 --- a/linters/amp_fix_comments.py +++ b/linters/amp_fix_comments.py @@ -8,6 +8,7 @@ import linters.amp_fix_comments as lamficom """ + import argparse import dataclasses import io @@ -248,7 +249,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", nargs="+", action="store", type=str, help="Files to process" diff --git a/linters/amp_fix_md_links.py b/linters/amp_fix_md_links.py index 3aad6261f..5d1021fe3 100644 --- a/linters/amp_fix_md_links.py +++ b/linters/amp_fix_md_links.py @@ -381,7 +381,6 @@ def fix_links(file_name: str) -> Tuple[List[str], List[str], List[str]]: class _LinkFixer(liaction.Action): - def check_if_possible(self) -> bool: return True @@ -399,7 +398,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_fix_whitespaces.py b/linters/amp_fix_whitespaces.py index 9076f78f8..2596f5864 100644 --- a/linters/amp_fix_whitespaces.py +++ b/linters/amp_fix_whitespaces.py @@ -104,7 +104,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_flake8.py b/linters/amp_flake8.py index 85bc4c053..309e58879 100644 --- a/linters/amp_flake8.py +++ b/linters/amp_flake8.py @@ -8,6 +8,7 @@ import linters.amp_flake8 as lampflak """ + import argparse import logging from typing import List @@ -112,7 +113,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_format_separating_line.py b/linters/amp_format_separating_line.py index 1069018bb..4222a09c1 100644 --- a/linters/amp_format_separating_line.py +++ b/linters/amp_format_separating_line.py @@ -75,7 +75,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_isort.py b/linters/amp_isort.py index b55b78bf0..152582a80 100644 --- a/linters/amp_isort.py +++ b/linters/amp_isort.py @@ -8,6 +8,7 @@ import linters.amp_isort as lampisor """ + import argparse import logging import os @@ -70,7 +71,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--abort_on_change", diff --git a/linters/amp_lint_md.py b/linters/amp_lint_md.py index 45594ba66..cd4a24a16 100644 --- a/linters/amp_lint_md.py +++ b/linters/amp_lint_md.py @@ -8,6 +8,7 @@ import linters.amp_lint_md as lamlimd """ + import argparse import logging import os @@ -40,11 +41,10 @@ def _check_readme_is_capitalized(file_name: str) -> str: class _LintMarkdown(liaction.Action): - def __init__(self) -> None: - #cmd = "find -wholename '*dev_scripts_helpers/documentation/lint_notes.py'" - #executable = hsystem.system_to_one_line(cmd) - #assert 0, executable + # cmd = "find -wholename '*dev_scripts_helpers/documentation/lint_notes.py'" + # executable = hsystem.system_to_one_line(cmd) + # assert 0, executable executable = "$(find -wholename '*dev_scripts_helpers/documentation/lint_notes.py')" super().__init__(executable) @@ -57,7 +57,9 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: ext = os.path.splitext(file_name)[1] output: List[str] = [] if ext != ".md": - _LOG.debug("Skipping file_name='%s' because ext='%s'", file_name, ext) + _LOG.debug( + "Skipping file_name='%s' because ext='%s'", file_name, ext + ) return output # Run lint_notes.py. cmd = [] @@ -65,7 +67,9 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: cmd.append(f"-i {file_name}") cmd.append(f"-o {file_name}") cmd_as_str = " ".join(cmd) - _, output = liutils.tee(cmd_as_str, self._executable, abort_on_error=True) + _, output = liutils.tee( + cmd_as_str, self._executable, abort_on_error=True + ) # Check file name. msg = _check_readme_is_capitalized(file_name) if msg: @@ -87,7 +91,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_mypy.py b/linters/amp_mypy.py index 2cbeaad0e..d193ea6a8 100644 --- a/linters/amp_mypy.py +++ b/linters/amp_mypy.py @@ -10,6 +10,7 @@ import linters.amp_mypy as lampmypy """ + import argparse import functools import logging @@ -58,7 +59,6 @@ def _ignore_misc_error(line: str, file_name: str) -> None: class _Mypy(liaction.Action): - def __init__(self) -> None: executable = "mypy" super().__init__(executable) @@ -131,7 +131,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_normalize_import.py b/linters/amp_normalize_import.py index 27221a205..05adf5fe3 100644 --- a/linters/amp_normalize_import.py +++ b/linters/amp_normalize_import.py @@ -158,7 +158,9 @@ def shorten_import_names(self, py_files: List[str]) -> LongImportToShort: _LOG.debug("# Processing '%s'", long_import) # E.g., "im.kitbot.data.load.kitbot_s3_data_loader" -> # "imkdalokis3datloa". - short_import = self._shorten_import(long_import, long_import_to_short) + short_import = self._shorten_import( + long_import, long_import_to_short + ) if short_import is not None: hdbg.dassert_ne(short_import, "") # Store the long import to short import mapping. @@ -231,7 +233,9 @@ def _use_special_abbreviations( return chunks_tmp, chunk_lengths_tmp @staticmethod - def _compute_short_import(chunks: Chunks, chunk_lengths: ChunkLengths) -> str: + def _compute_short_import( + chunks: Chunks, chunk_lengths: ChunkLengths + ) -> str: """ Compute the short import from the chunks and the desired length for each chunk. @@ -304,7 +308,8 @@ def _compute_max_chunk_lengths( # [1, 3, 4] <- weighted chunk lengths # [1, 3, 2] <- adj chunk lengths adj_chunk_lengths = [ - min(lengths) for lengths in zip(chunk_lengths, weighted_chunk_lengths) + min(lengths) + for lengths in zip(chunk_lengths, weighted_chunk_lengths) ] hdbg.dassert_eq(len(adj_chunk_lengths), len(chunks)) _LOG.debug("-> chunk_lengths_tmp=%s", str(adj_chunk_lengths)) @@ -595,7 +600,9 @@ def replace_short_imports_in_file( # Make sure that there are no conflicts of the new import with the existing # file code. conflict = self._is_short_import_used( - code=code, short_import=new_short_import, long_import=long_import + code=code, + short_import=new_short_import, + long_import=long_import, ) hdbg.dassert( not conflict, @@ -665,7 +672,9 @@ def _replace(matchobj: re.Match) -> str: # Do not replace an import if it is inside a string. res: str = matchobj.group() return res - res = matchobj.group().replace(old_short_import, new_short_import) + res = matchobj.group().replace( + old_short_import, new_short_import + ) return res # Replace the short import in the imports section of the file. @@ -1058,7 +1067,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--generate_map", diff --git a/linters/amp_processjupytext.py b/linters/amp_processjupytext.py index 547f5b364..46814cbb8 100644 --- a/linters/amp_processjupytext.py +++ b/linters/amp_processjupytext.py @@ -8,6 +8,7 @@ import linters.amp_processjupytext as lampproc """ + import argparse import logging from typing import List @@ -30,7 +31,6 @@ class _ProcessJupytext(liaction.Action): - def __init__(self, jupytext_action: str) -> None: executable = "$(find -wholename '*dev_scripts_helpers/notebooks/process_jupytext.py')" super().__init__(executable) @@ -65,7 +65,6 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: class _SyncJupytext(_ProcessJupytext): - def __init__(self) -> None: super().__init__("sync") @@ -76,7 +75,6 @@ def __init__(self) -> None: class _TestJupytext(_ProcessJupytext): - def __init__(self) -> None: super().__init__("test") @@ -100,7 +98,8 @@ def __init__(self) -> None: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_pylint.py b/linters/amp_pylint.py index 1c112c015..fe2eae10c 100644 --- a/linters/amp_pylint.py +++ b/linters/amp_pylint.py @@ -8,6 +8,7 @@ import linters.amp_pylint as lamppyli """ + import argparse import logging import os @@ -31,7 +32,6 @@ class _Pylint(liaction.Action): - def __init__(self) -> None: executable = "pylint" super().__init__(executable) @@ -211,7 +211,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_remove_empty_lines_in_function.py b/linters/amp_remove_empty_lines_in_function.py index 7f1150c47..67734b8b1 100644 --- a/linters/amp_remove_empty_lines_in_function.py +++ b/linters/amp_remove_empty_lines_in_function.py @@ -6,6 +6,7 @@ import linters.amp_remove_empty_lines_in_function as larelinfu """ + import argparse import logging import re @@ -53,7 +54,9 @@ def _remove_empty_lines(text: str) -> List[str]: continue if inside_function and stripped == "": # Remove empty lines inside the function. - _LOG.debug("Removing empty line found at line %d inside function.", i) + _LOG.debug( + "Removing empty line found at line %d inside function.", i + ) continue if inside_function and stripped != "" and current_indent <= base_indent: # Retain trailing empty lines after the function, @@ -89,7 +92,6 @@ def _remove_empty_lines(text: str) -> List[str]: class _RemoveEmptyLines(liaction.Action): - def check_if_possible(self) -> bool: return True @@ -110,7 +112,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/amp_warn_incorrectly_formatted_todo.py b/linters/amp_warn_incorrectly_formatted_todo.py index ba6464ff6..aadf25616 100644 --- a/linters/amp_warn_incorrectly_formatted_todo.py +++ b/linters/amp_warn_incorrectly_formatted_todo.py @@ -73,7 +73,8 @@ def _execute(self, file_name: str, pedantic: int) -> List[str]: def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "files", diff --git a/linters/base.py b/linters/base.py index f24cfeb28..0f3b8fec6 100755 --- a/linters/base.py +++ b/linters/base.py @@ -285,7 +285,9 @@ def _lint( cur_action_lints = action_class.execute(file_path, pedantic) hdbg.dassert_list_of_strings(cur_action_lints) # Annotate each lint with a [tag] specifying the action name. - cur_action_lints = [lnt + f" [{action_name}]" for lnt in cur_action_lints] + cur_action_lints = [ + lnt + f" [{action_name}]" for lnt in cur_action_lints + ] lints.extend(cur_action_lints) in_tmp_scratch_dir = liutils.is_under_tmp_scratch_dir(file_path) if not hserver.is_inside_ci() and not in_tmp_scratch_dir: @@ -349,7 +351,8 @@ def _run_linter( def _parse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) # File selection. parser.add_argument( diff --git a/linters/test/test_amp_check_import.py b/linters/test/test_amp_check_import.py index 5b6ee5c45..0d1da5956 100644 --- a/linters/test/test_amp_check_import.py +++ b/linters/test/test_amp_check_import.py @@ -17,7 +17,7 @@ def test2(self) -> None: Test from lib import something: invalid. """ line = "from pandas import DataFrame" - exp = f"do not use '{line}' use 'import foo.bar " "as fba'" + exp = f"do not use '{line}' use 'import foo.bar as fba'" self._helper_check_import(line, exp, file_name="test.py") def test3(self) -> None: diff --git a/linters/test/test_amp_check_md_reference.py b/linters/test/test_amp_check_md_reference.py index d1c2627db..17aeedbd8 100644 --- a/linters/test/test_amp_check_md_reference.py +++ b/linters/test/test_amp_check_md_reference.py @@ -14,7 +14,6 @@ class Test_check_file_reference(hunitest.TestCase): - def test1(self) -> None: """ Test for a referenced Markdown file in README.md. @@ -35,7 +34,9 @@ def test2(self) -> None: warnings = lachmdre.check_file_reference(readme_path, "unreferenced.md") self.assertEqual( warnings, - ["unreferenced.md: 'unreferenced.md' is not referenced in README.md"], + [ + "unreferenced.md: 'unreferenced.md' is not referenced in README.md" + ], ) def _create_tmp_dir_with_file(self) -> str: diff --git a/linters/test/test_amp_check_md_toc_headers.py b/linters/test/test_amp_check_md_toc_headers.py index 633364c30..64ef6bbbc 100644 --- a/linters/test/test_amp_check_md_toc_headers.py +++ b/linters/test/test_amp_check_md_toc_headers.py @@ -14,7 +14,6 @@ class Test_fix_md_headers(hunitest.TestCase): - def test1(self) -> None: """ Test that no modifications are made when headers are correct. @@ -153,7 +152,6 @@ def _write_input_file(self, txt: str, file_name: str) -> str: class Test_verify_toc_postion(hunitest.TestCase): - def test1(self) -> None: """ Test that a warning is issued when content appears before TOC. diff --git a/linters/test/test_amp_dev_scripts.py b/linters/test/test_amp_dev_scripts.py index 03fc4c112..a6158f7c9 100644 --- a/linters/test/test_amp_dev_scripts.py +++ b/linters/test/test_amp_dev_scripts.py @@ -28,7 +28,6 @@ class Test_linter_py1(hunitest.TestCase): - def write_input_file(self, txt: str, file_name: str) -> Tuple[str, str]: """ Write test content to the file. diff --git a/linters/test/test_amp_doc_formatter.py b/linters/test/test_amp_doc_formatter.py index fa71c10ef..d954586fc 100644 --- a/linters/test/test_amp_doc_formatter.py +++ b/linters/test/test_amp_doc_formatter.py @@ -12,7 +12,6 @@ class Test_docformatter(hunitest.TestCase): - def test1(self) -> None: """ Test that the docstring should be dedented. @@ -185,7 +184,6 @@ def _docformatter(self, text: str) -> Tuple[str, List[str], str]: class TestFindUnbalancedBackticks(hunitest.TestCase): - def test1(self) -> None: """ Test that the starting indices of docstrings with unbalanced backticks diff --git a/linters/test/test_amp_fix_comment.py b/linters/test/test_amp_fix_comment.py index 6219c3a0f..5dc1b52e1 100644 --- a/linters/test/test_amp_fix_comment.py +++ b/linters/test/test_amp_fix_comment.py @@ -406,9 +406,7 @@ def test1(self) -> None: test.method_after() # - a list item # - another list item - """.split( - "\n" - ) + """.split("\n") expected = """ test.method_before() # This is a super long message that has too much information in it. Although @@ -416,9 +414,7 @@ def test1(self) -> None: test.method_after() # - a list item # - another list item - """.split( - "\n" - ) + """.split("\n") actual = lamficom._reflow_comments_in_lines(lines=content) self.assertEqual(expected, actual) diff --git a/linters/test/test_amp_fix_md_links.py b/linters/test/test_amp_fix_md_links.py index 64de5c7e7..e25eeb038 100644 --- a/linters/test/test_amp_fix_md_links.py +++ b/linters/test/test_amp_fix_md_links.py @@ -37,8 +37,9 @@ def _get_output_string(out_warnings: List[str], updated_lines: List[str]) -> str class Test_fix_links(hunitest.TestCase): - - def run_test(self, input_content: str, *, file_name: str = "test.md") -> None: + def run_test( + self, input_content: str, *, file_name: str = "test.md" + ) -> None: """ Helper method to run tests with common setup and verification. @@ -203,7 +204,9 @@ def test5(self) -> None: Markdown link: [Valid Markdown and header Link]({reference_file_link}#hyphen-test) """ - self.run_test(reference_file_md_content, file_name="valid_header_test.md") + self.run_test( + reference_file_md_content, file_name="valid_header_test.md" + ) def test6(self) -> None: """ @@ -508,7 +511,6 @@ def test19(self) -> None: class Test_make_path_absolute(hunitest.TestCase): - def test_make_path_absolute1(self) -> None: """ Test file path to retain directory name beginning with a dot. diff --git a/linters/test/test_amp_normalize_import.py b/linters/test/test_amp_normalize_import.py index 9aef4be9c..64035a858 100644 --- a/linters/test/test_amp_normalize_import.py +++ b/linters/test/test_amp_normalize_import.py @@ -21,7 +21,6 @@ class TestChunkify(hunitest.TestCase): - def test1(self) -> None: """ Test long imports separated by dots only. @@ -47,7 +46,6 @@ def test2(self) -> None: class TestComputeMaxChunkLengths(hunitest.TestCase): - def test1(self) -> None: """ Test one chunk, given that length <= 8. @@ -99,7 +97,6 @@ def test4(self) -> None: class TestComputeShortImport(hunitest.TestCase): - def test1(self) -> None: """ Test short import computation. @@ -131,7 +128,6 @@ def test2(self) -> None: class TestUseSpecialAbbreviations(hunitest.TestCase): - def test1(self) -> None: """ Test "helpers" (as a dir name). @@ -187,7 +183,6 @@ def test4(self) -> None: class TestSearchForUniqueShortImport(hunitest.TestCase): - def test1(self) -> None: """ Test search given that short import is not in existing short imports. @@ -242,7 +237,6 @@ def test3(self) -> None: class TestShortenImport(hunitest.TestCase): - def test1(self) -> None: """ Test shorten import for long helper name (>7 chars). @@ -310,7 +304,6 @@ def test6(self) -> None: class TestFindCollisions(hunitest.TestCase): - def test1(self) -> None: """ Test no collisions. @@ -344,7 +337,6 @@ def test2(self) -> None: class TestGetLongImportFromFilePath(hunitest.TestCase): - def test1(self) -> None: """ Test that "py" letters inside a file name are not removed. @@ -432,7 +424,6 @@ def test6(self) -> None: class TestShortenImportNames(hunitest.TestCase): - def test_end_to_end1(self) -> None: """ Test shorten import end to end. @@ -532,7 +523,6 @@ def test_end_to_end2(self) -> None: class TestIsShortImportUsed(hunitest.TestCase): - def test1(self) -> None: """ Short import is used with the correct long import. @@ -685,7 +675,6 @@ def test10(self) -> None: class TestExtractExistingImportMappingsFromCode(hunitest.TestCase): - def test1(self) -> None: """ Test an import without a short import. @@ -694,9 +683,7 @@ def test1(self) -> None: expected: List[Tuple[str, str]] = [] act = lamnoimp.CodeImportNormalizer._extract_existing_import_mappings_from_code( code=code - )[ - 0 - ] + )[0] self.assertEqual(act, expected) def test2(self) -> None: @@ -707,9 +694,7 @@ def test2(self) -> None: expected = [("test", "te")] act = lamnoimp.CodeImportNormalizer._extract_existing_import_mappings_from_code( code=code - )[ - 0 - ] + )[0] self.assertEqual(act, expected) def test3(self) -> None: @@ -720,9 +705,7 @@ def test3(self) -> None: expected = [("test.sub", "tsub")] act = lamnoimp.CodeImportNormalizer._extract_existing_import_mappings_from_code( code=code - )[ - 0 - ] + )[0] self.assertEqual(act, expected) def test4(self) -> None: @@ -736,9 +719,7 @@ def test4(self) -> None: expected = [("test", "te"), ("test.sub", "tsub")] act = lamnoimp.CodeImportNormalizer._extract_existing_import_mappings_from_code( code=code - )[ - 0 - ] + )[0] self.assertEqual(act, expected) def test5(self) -> None: @@ -751,9 +732,7 @@ def test5(self) -> None: expected: List[Tuple[str, str]] = [] act = lamnoimp.CodeImportNormalizer._extract_existing_import_mappings_from_code( code=code - )[ - 0 - ] + )[0] self.assertEqual(act, expected) def test6(self) -> None: @@ -773,9 +752,7 @@ def func_xyz(...): expected = [("helpers.abc", "abc"), ("helpers.abc", "xyz")] act = lamnoimp.CodeImportNormalizer._extract_existing_import_mappings_from_code( code=code - )[ - 0 - ] + )[0] self.assertEqual(act, expected) @@ -785,7 +762,6 @@ def func_xyz(...): class TestReplaceShortImportInCode(hunitest.TestCase): - def test1(self) -> None: """ No matches. @@ -967,7 +943,6 @@ def _helper(self, code: str, expected: str) -> None: class TestReplaceShortImportsInFile(hunitest.TestCase): - def test1(self) -> None: """ Test local package wrong short import. @@ -1130,7 +1105,6 @@ def _check_import_normalization(self, code: str, expected_code: str) -> None: class TestInsertDocstringIfNeeded(hunitest.TestCase): - def test1(self) -> None: """ Test that the import docstring is not added. @@ -1247,7 +1221,6 @@ def _helper( class TestCleanDocstringFromImport(hunitest.TestCase): - def test1(self) -> None: """ Test no import line in the docstring text. @@ -1320,7 +1293,6 @@ def _helper( class TestProcessContent(hunitest.TestCase): - def test1(self) -> None: """ Test that the import line is added. @@ -1529,7 +1501,6 @@ def _helper( class TestEndToEnd(hunitest.TestCase): - def test(self) -> None: """ Test that import normalization works end-to-end. diff --git a/linters/test/test_amp_remove_empty_lines_in_function.py b/linters/test/test_amp_remove_empty_lines_in_function.py index 50defbd2b..dad1b83d9 100644 --- a/linters/test/test_amp_remove_empty_lines_in_function.py +++ b/linters/test/test_amp_remove_empty_lines_in_function.py @@ -11,7 +11,6 @@ class Test_remove_empty_lines(hunitest.TestCase): - def test1(self) -> None: """ Test cleaning empty lines in a single function. diff --git a/linters/utils.py b/linters/utils.py index 6aa787c41..ef33b5cb8 100644 --- a/linters/utils.py +++ b/linters/utils.py @@ -191,7 +191,9 @@ def write_file_back( # TODO(saggese): should this be moved to system interactions? -def tee(cmd: str, executable: str, abort_on_error: bool) -> Tuple[int, List[str]]: +def tee( + cmd: str, executable: str, abort_on_error: bool +) -> Tuple[int, List[str]]: """ Execute command "cmd", capturing its output and removing empty lines. diff --git a/notebook_template.ipynb b/notebook_template.ipynb index 050d87daf..448991546 100644 --- a/notebook_template.ipynb +++ b/notebook_template.ipynb @@ -43,7 +43,7 @@ "%autoreload 2\n", "\n", "import logging\n", - " \n", + "\n", "import helpers.hdbg as hdbg\n", "import helpers.henv as henv\n", "import helpers.hprint as hprint" diff --git a/notebook_template.py b/notebook_template.py index 1b39dfbcc..19effd0fa 100644 --- a/notebook_template.py +++ b/notebook_template.py @@ -32,7 +32,7 @@ # %autoreload 2 import logging - + import helpers.hdbg as hdbg import helpers.henv as henv import helpers.hprint as hprint diff --git a/pyproject.toml b/pyproject.toml index 3d8890ac2..5305529b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.ruff] -line-length = 80 +line-length = 81 target-version = "py311" fix = true exclude = [ diff --git a/template_unit_test.py b/template_unit_test.py index d599480a1..4fc6ae256 100644 --- a/template_unit_test.py +++ b/template_unit_test.py @@ -12,7 +12,6 @@ class Test_format_compressed_markdown1(hunitest.TestCase): - def test1(self) -> None: # Prepare inputs. # ... From 61e00b6970b6184d94e31b68889ffc4cc8d3291c Mon Sep 17 00:00:00 2001 From: GP Saggese Date: Tue, 3 Jun 2025 09:26:58 -0400 Subject: [PATCH 3/3] Lint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-commit checks: All checks passed ✅ --- ...Task2291_Playback_approach_for_unit_testing.ipynb | 1 - pyproject.toml | 12 +++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/helpers/notebooks/PartTask2291_Playback_approach_for_unit_testing.ipynb b/helpers/notebooks/PartTask2291_Playback_approach_for_unit_testing.ipynb index d38079e6b..bb04dd030 100644 --- a/helpers/notebooks/PartTask2291_Playback_approach_for_unit_testing.ipynb +++ b/helpers/notebooks/PartTask2291_Playback_approach_for_unit_testing.ipynb @@ -129,7 +129,6 @@ } ], "source": [ - "import pandas as pd\n", "\n", "data = {\n", " \"Product\": [\"Desktop Computer\", \"Tablet\", \"iPhone\", \"Laptop\"],\n", diff --git a/pyproject.toml b/pyproject.toml index 5305529b0..7e1480c2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,12 +6,11 @@ exclude = [ "**/outcomes/**", "**/import_check/example/**" ] +output-format = "concise" [tool.ruff.lint] -exclude = [ - "**/outcomes/**", - "**/import_check/example/**" -] +# E731 Do not assign a `lambda` expression, use a `def` +ignore = ["E731"] [tool.pyrefly] # Currently, Pyrefly doesn't officially support [tool.pyrefly] config @@ -23,5 +22,8 @@ path = "." lint.ignore_codes = [] lint.enabled = true lint.format = "text" -lint.exclude = ["tests/fixtures"] +lint.exclude = [ + "**/outcomes/**", + "**/import_check/example/**" +] lint.show_source = true