From 3da5ddcaac382086554aa80b7129ee2ddbd9e1f0 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 9 Dec 2025 15:01:18 +0100 Subject: [PATCH 1/8] feat: add registry creation --- julio/__init__.pyi | 5 ++ julio/_cli.py | 154 ++++++++++++++++++++++++++++++++++++++++ julio/_functions.py | 56 +++++++++++++++ pyproject.toml | 2 + tests/__init__.py | 0 tests/test_functions.py | 29 ++++++++ uv.lock | 14 ++++ 7 files changed, 260 insertions(+) create mode 100644 julio/_cli.py create mode 100644 julio/_functions.py create mode 100644 tests/__init__.py create mode 100644 tests/test_functions.py diff --git a/julio/__init__.pyi b/julio/__init__.pyi index e69de29..575fb65 100644 --- a/julio/__init__.pyi +++ b/julio/__init__.pyi @@ -0,0 +1,5 @@ +__all__ = [ + "create", +] + +from ._functions import create diff --git a/julio/_cli.py b/julio/_cli.py new file mode 100644 index 0000000..42a922e --- /dev/null +++ b/julio/_cli.py @@ -0,0 +1,154 @@ +"""CLI for julio.""" + +# Authors: Synchon Mandal +# License: AGPL + +import logging +import logging.config +import pathlib +import sys + +import click +import datalad +import structlog + +from . import _functions as cli_func + + +__all__ = ["cli", "create"] + +# Common processors for stdlib and structlog +_timestamper = structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S") + + +def _remove_datalad_message(_, __, event_dict): + """Clean datalad records.""" + if "message" in event_dict: + event_dict.pop("message") + if "dlm_progress" in event_dict: + event_dict.pop("dlm_progress") + if "dlm_progress_noninteractive_level" in event_dict: + event_dict.pop("dlm_progress_noninteractive_level") + if "dlm_progress_update" in event_dict: + event_dict.pop("dlm_progress_update") + if "dlm_progress_label" in event_dict: + event_dict.pop("dlm_progress_label") + if "dlm_progress_unit" in event_dict: + event_dict.pop("dlm_progress_unit") + if "dlm_progress_total" in event_dict: + event_dict.pop("dlm_progress_total") + return event_dict + + +_pre_chain = [ + structlog.stdlib.add_log_level, + structlog.stdlib.add_logger_name, + structlog.stdlib.ExtraAdder(), + _timestamper, + _remove_datalad_message, +] + + +def _set_log_config(verbose: int) -> None: + """Set logging config. + + Parameters + ---------- + verbose : int + Verbosity. + + """ + # Configure logger based on verbosity + if verbose == 0: + level = logging.WARNING + elif verbose == 1: + level = logging.INFO + else: + level = logging.DEBUG + # Configure stdlib + logging.config.dictConfig( + { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "console": { + "()": structlog.stdlib.ProcessorFormatter, + "processors": [ + structlog.stdlib.add_logger_name, + structlog.stdlib.ProcessorFormatter.remove_processors_meta, + structlog.dev.ConsoleRenderer( + colors=sys.stdout.isatty() and sys.stderr.isatty() + ), + ], + "foreign_pre_chain": _pre_chain, + }, + }, + "handlers": { + "default": { + "level": level, + "class": "logging.StreamHandler", + "formatter": "console", + }, + }, + "loggers": { + "": { + "handlers": ["default"], + "level": level, + "propagate": True, + }, + }, + } + ) + # Configure structlog + structlog.configure( + processors=[ + structlog.stdlib.add_log_level, + structlog.stdlib.PositionalArgumentsFormatter(), + structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S"), + structlog.processors.StackInfoRenderer(), + structlog.processors.format_exc_info, + structlog.stdlib.ProcessorFormatter.wrap_for_formatter, + ], + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.make_filtering_bound_logger(level), + cache_logger_on_first_use=True, + ) + # Remove datalad logger handlers to avoid duplicate logging + _datalad_lgr_hdlrs = datalad.log.lgr.handlers + for h in _datalad_lgr_hdlrs: + datalad.log.lgr.removeHandler(h) + datalad.log.lgr.setLevel(level) + + +@click.group +@click.version_option(prog_name="julio") +@click.help_option() +def cli() -> None: + """julio CLI.""" # noqa: D403 + + +@cli.command +@click.argument( + "registry_path", + type=click.Path( + exists=False, + readable=True, + writable=True, + file_okay=False, + path_type=pathlib.Path, + ), + metavar="", +) +@click.option("-v", "--verbose", count=True, type=int) +def create( + registry_path: click.Path, + verbose: int, +) -> None: + """Create registry.""" + _set_log_config(verbose) + try: + cli_func.create(registry_path) + except RuntimeError as err: + click.echo(f"{err}", err=True) + else: + click.echo("Success") diff --git a/julio/_functions.py b/julio/_functions.py new file mode 100644 index 0000000..8f5ce90 --- /dev/null +++ b/julio/_functions.py @@ -0,0 +1,56 @@ +"""Functions for julio.""" + +# Authors: Synchon Mandal +# License: AGPL + +from pathlib import Path + +import datalad.api as dl +import structlog +from datalad.support.exceptions import IncompleteResultsError + + +__all__ = ["create"] + + +logger = structlog.get_logger() + + +def create(registry_path: Path): + """Create a registry at `registry_path`. + + Parameters + ---------- + registry_path : pathlib.Path + Path to the registry. + + Raises + ------ + RuntimeError + If there is a problem creating the registry. + + """ + try: + ds = dl.create( + path=registry_path, + cfg_proc="text2git", + on_failure="stop", + result_renderer="disabled", + ) + except IncompleteResultsError as e: + raise RuntimeError(f"Failed to create dataset: {e.failed}") from e + else: + logger.debug( + "Registry created successfully", + cmd="create", + path=str(registry_path.resolve()), + ) + # Add config file + conf_path = Path(ds.path) / "registry-config.yml" + conf_path.touch() + ds.save( + conf_path, + message="[julio] add registry configuration", + on_failure="stop", + result_renderer="disabled", + ) diff --git a/pyproject.toml b/pyproject.toml index c09fa5d..bcfb5c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "click>=8.1.3,<8.2", "datalad>=1.0.0,<1.3.0", "lazy_loader==0.4", + "structlog>=25.0.0,<26.0.0", ] dynamic = ["version"] @@ -123,6 +124,7 @@ known-first-party = ["julio"] known-third-party =[ "click", "datalad", + "structlog", "pytest", ] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_functions.py b/tests/test_functions.py new file mode 100644 index 0000000..978fb8a --- /dev/null +++ b/tests/test_functions.py @@ -0,0 +1,29 @@ +"""Test for functions.""" + +# Authors: Synchon Mandal +# License: AGPL + +import shutil +from pathlib import Path + +import pytest + +from julio import create + + +def test_create(tmp_path: Path) -> None: + """Test registry creation. + + Parameters + ---------- + tmp_path : Path + Pytest fixture that provides a temporary directory. + + """ + registry_path = tmp_path / "test_registry" + create(registry_path) + config_path = registry_path / "registry-config.yml" + assert config_path.is_file() + with pytest.raises(RuntimeError): + create(registry_path) + shutil.rmtree(registry_path) diff --git a/uv.lock b/uv.lock index dcff20f..8b60db2 100644 --- a/uv.lock +++ b/uv.lock @@ -488,6 +488,7 @@ dependencies = [ { name = "click" }, { name = "datalad" }, { name = "lazy-loader" }, + { name = "structlog" }, ] [package.optional-dependencies] @@ -505,6 +506,7 @@ requires-dist = [ { name = "lazy-loader", specifier = "==0.4" }, { name = "pre-commit", marker = "extra == 'dev'" }, { name = "ruff", marker = "extra == 'dev'" }, + { name = "structlog", specifier = ">=25.0.0,<26.0.0" }, { name = "towncrier", marker = "extra == 'dev'" }, { name = "tox", marker = "extra == 'dev'" }, ] @@ -1000,6 +1002,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "structlog" +version = "25.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/52/9ba0f43b686e7f3ddfeaa78ac3af750292662284b3661e91ad5494f21dbc/structlog-25.5.0.tar.gz", hash = "sha256:098522a3bebed9153d4570c6d0288abf80a031dfdb2048d59a49e9dc2190fc98", size = 1460830, upload-time = "2025-10-27T08:28:23.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/45/a132b9074aa18e799b891b91ad72133c98d8042c70f6240e4c5f9dabee2f/structlog-25.5.0-py3-none-any.whl", hash = "sha256:a8453e9b9e636ec59bd9e79bbd4a72f025981b3ba0f5837aebf48f02f37a7f9f", size = 72510, upload-time = "2025-10-27T08:28:21.535Z" }, +] + [[package]] name = "tomli" version = "2.3.0" From aa3f05551018f0de8c7623a9846e9d0e50d6deda Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 9 Dec 2025 15:01:45 +0100 Subject: [PATCH 2/8] chore: update tox.ini for coverage --- tox.ini | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index abf06aa..034e1dc 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ requires = env_list = ruff, changelog, - test, + coverage, py3{10,11,12,13,14} isolated_build = true @@ -13,11 +13,11 @@ python = 3.10: py310 3.11: py311 3.12: py312 - 3.13: py313 + 3.13: coverage 3.14: py314 [testenv] -skip_install = false +skip_install = true passenv = HOME deps = @@ -42,3 +42,36 @@ deps = lazy_loader==0.4 commands = towncrier build --draft + +[testenv:coverage] +description = run tests with coverage +skip_install = true +deps = + pytest-cov +commands = + pytest --cov={toxinidir}/julio \ + --cov-report=xml \ + --cov-report=term \ + {toxinidir}/tests + +[coverage:paths] +source = + tests + +[coverage:run] +branch = true +omit = + */_version.py + */__init__.py + */_cli.py +parallel = false + +[coverage:report] +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + # Type checking if statements should not be considered + if TYPE_CHECKING: + # Don't complain if non-runnable code isn't run: + if __name__ == .__main__.: +precision = 2 From 9e352b5bfe1995b7e3980d54d77be9897fc0364e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 9 Dec 2025 15:04:04 +0100 Subject: [PATCH 3/8] chore(ci): add ci.yml --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1f0f4b1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: Run Tests + +on: + - push + - pull_request + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" + + steps: + - uses: actions/checkout@v6 + - name: Set up git-annex + run: | + sudo apt-get update + sudo apt-get install -y git-annex + - name: Install Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install tox tox-gh-actions + - name: Configure Git for DataLad + run: | + git config --global user.email "julio-runner@github.com" + git config --global user.name "GitHub Runner" + - name: Test with tox + run: | + tox From ab0031d25991078594c9d38113080e795312e3af Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 11 Dec 2025 07:46:35 +0100 Subject: [PATCH 4/8] chore(tests): move tests inside source and update coverage config --- {tests => julio/tests}/test_functions.py | 0 pyproject.toml | 27 ++++++++++++++++++++- tests/__init__.py | 0 tox.ini | 31 +++--------------------- 4 files changed, 29 insertions(+), 29 deletions(-) rename {tests => julio/tests}/test_functions.py (100%) delete mode 100644 tests/__init__.py diff --git a/tests/test_functions.py b/julio/tests/test_functions.py similarity index 100% rename from tests/test_functions.py rename to julio/tests/test_functions.py diff --git a/pyproject.toml b/pyproject.toml index bcfb5c3..581c77d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -130,7 +130,7 @@ known-third-party =[ [tool.pytest.ini_options] minversion = "7.0" -testpaths = "tests" +testpaths = "julio" log_cli_level = "INFO" xfail_strict = true addopts = [ @@ -178,3 +178,28 @@ showcontent = true directory = "fixed" name = "Fixed" showcontent = true + +[tool.coverage.paths] +source = [ + "julio", + "*/site-packages/julio", +] + +[tool.coverage.run] +branch = true +omit = [ + "*/_version.py", + "*/_cli.py", + "*/tests/*", +] + +[tool.coverage.report] +exclude_lines = [ + # Have to re-enable the standard pragma + "pragma: no cover", + # Type checking if statements should not be considered + "if TYPE_CHECKING:", + # Don't complain if non-runnable code isn't run: + "if __name__ == .__main__.:", +] +precision = 2 diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tox.ini b/tox.ini index 034e1dc..f46a1e7 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ python = 3.14: py314 [testenv] -skip_install = true +skip_install = false passenv = HOME deps = @@ -45,33 +45,8 @@ commands = [testenv:coverage] description = run tests with coverage -skip_install = true +skip_install = false deps = pytest-cov commands = - pytest --cov={toxinidir}/julio \ - --cov-report=xml \ - --cov-report=term \ - {toxinidir}/tests - -[coverage:paths] -source = - tests - -[coverage:run] -branch = true -omit = - */_version.py - */__init__.py - */_cli.py -parallel = false - -[coverage:report] -exclude_lines = - # Have to re-enable the standard pragma - pragma: no cover - # Type checking if statements should not be considered - if TYPE_CHECKING: - # Don't complain if non-runnable code isn't run: - if __name__ == .__main__.: -precision = 2 + pytest --cov={envsitepackagesdir}/julio --cov-report=xml --cov-report=term --cov-config=pyproject.toml {envsitepackagesdir}/julio From 870b9644070fb4fd877b049303a52b85721fd953 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 11 Dec 2025 07:48:03 +0100 Subject: [PATCH 5/8] chore: update coverage omit --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 581c77d..7fe3fda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -188,6 +188,7 @@ source = [ [tool.coverage.run] branch = true omit = [ + "*/__init__.py", "*/_version.py", "*/_cli.py", "*/tests/*", From b75467551273955c3ceee3feaf836667d95a37bf Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 11 Dec 2025 07:53:43 +0100 Subject: [PATCH 6/8] chore(ci): add codecov config --- .github/workflows/ci.yml | 6 ++++++ codecov.yml | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 codecov.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f0f4b1..d5ccc42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,3 +38,9 @@ jobs: - name: Test with tox run: | tox + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + fail_ci_if_error: true + flags: julio + if: success() && matrix.python-version == 3.13 diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..d080813 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,29 @@ +codecov: + notify: {} + require_ci_to_pass: false + +comment: # this is a top-level key + layout: "reach, diff, flags, files" + behavior: default + require_changes: false # if true: only post the comment if coverage changes + require_base: no # [yes :: must have a base report to post] + +coverage: + status: + project: off + patch: off + +flag_management: + default_rules: + carryforward: true + individual_flags: + - name: julio + paths: + - julio/ + statuses: + - type: project + target: 90% + threshold: 1% + - type: patch + target: 95% + threshold: 1% From 492d1cca8082add73944dab5a3e1f88a59035f30 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 11 Dec 2025 18:17:44 +0100 Subject: [PATCH 7/8] chore(log): update CHANGELOG.md --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a1bff0..5c13509 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [0.0.1](https://github.com/juaml/juni-julio/tree/0.0.1) - 2025-12-11 + +### Added + +- Introduce registry creation ([#2](https://github.com/juaml/juni-julio/issues/2)) + + ## [0.0.0](https://github.com/juaml/junifer-julio/tree/0.0.0) - 2025-12-11 ### Added From 38e8c574a0628ec6c2e187c100f9c40dfcd4e8fd Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 11 Dec 2025 18:18:37 +0100 Subject: [PATCH 8/8] chore: rename repo to junifer-julio --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c13509..364bff6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## [0.0.1](https://github.com/juaml/juni-julio/tree/0.0.1) - 2025-12-11 +## [0.0.1](https://github.com/juaml/junifer-julio/tree/0.0.1) - 2025-12-11 ### Added