Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 52 additions & 1 deletion auto_dev/commands/repo.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,17 @@
import os
import sys
import difflib
import subprocess
from shutil import rmtree
from pathlib import Path
from dataclasses import dataclass

import toml
import requests
import rich_click as click
from rich import print # pylint: disable=W0622
from toml import TomlPreserveInlineDictEncoder
from jinja2 import Environment, FileSystemLoader
from rich.prompt import Prompt
from rich.progress import Progress, track
from aea.cli.utils.config import get_default_author_from_cli_config
Expand All @@ -26,9 +30,11 @@
from auto_dev.enums import UserInput
from auto_dev.utils import change_dir
from auto_dev.constants import (
THIS_REPO_ROOT,
DEFAULT_TIMEOUT,
TEMPLATE_FOLDER,
DEFAULT_ENCODING,
JINJA_TEMPLATE_FOLDER,
SAMPLE_PYTHON_CLI_FILE,
SAMPLE_PYTHON_MAIN_FILE,
CheckResult,
Expand Down Expand Up @@ -60,6 +66,34 @@ def execute_commands(*commands: str, verbose: bool, logger, shell: bool = False)
sys.exit(1)


def _render_autonomy_pyproject_template(project_name: str, authors: str) -> str:
env = Environment(loader=FileSystemLoader(JINJA_TEMPLATE_FOLDER), autoescape=False) # noqa
template = env.get_template("repo/autonomy/pyproject.jinja")

pyproject_path = THIS_REPO_ROOT / "pyproject.toml"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

consider a more minimal pyproject than adev's for scaffoled repos. (e.g. dev deps in adev pyproject include mkdocs, and may not be appropriate for scaffolded repos)

pyproject = toml.load(pyproject_path)

build_system = pyproject["build-system"]
classifiers = pyproject["tool"]["poetry"]["classifiers"]
python = pyproject["tool"]["poetry"]["dependencies"]["python"]
current_version = pyproject["tool"]["poetry"]["version"]
min_minor_version = ".".join(current_version.split(".")[:2])
version = f">={min_minor_version}.0,<={current_version}"
dev_dependencies = pyproject["tool"]["poetry"]["group"]["dev"]["dependencies"]
black_config = pyproject["tool"]["black"]

return template.render(
build_system=toml.dumps(build_system),
project_name=project_name,
authors=authors,
classifiers=" " + ",\n ".join(f'"{c}"' for c in classifiers),
python=python,
version=version,
dev_dependencies=toml.dumps(dev_dependencies, TomlPreserveInlineDictEncoder()),
black_config=toml.dumps(black_config),
)
Comment on lines +69 to +94
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Previously, updating dependencies in adev required a manual two-step process to keep the autonomy scaffold in sync:

  1. You had to update pyproject.toml.template in autonomy whenever a dependency or python version range changed in adev.
  2. You then had to regenerate the poetry.lock file after rendering the template, to ensure it matched the updated dependencies. If you didn’t, poetry install would fail, and the corresponding test would break.

While these failures correctly caught drift between the scaffolded repo and adev, the workflow was tedious. On top of that, the .template still hardcoded things like the adev version and tool configurations, and hence only partial consistency was attained.

This PR removes the need for all that: the autonomy pyproject.toml is now rendered from a jinja template that pulls version ranges, tool settings, and dependencies directly from adev's pyproject.toml. As a result, we no longer need to separately verify that the scaffolded repo matches adev: they're guaranteed to be in sync by construction.



cli = build_cli()

render_args = {
Expand Down Expand Up @@ -102,6 +136,23 @@ def scaffold(
"""Scaffold files for a new repo."""
new_repo_dir = Path.cwd()
template_folder = TEMPLATES[self.type_of_repo]

if self.type_of_repo == "autonomy":
project_name = self.scaffold_kwargs["project_name"]
authors = self.scaffold_kwargs["author"]
pyproject_content = _render_autonomy_pyproject_template(project_name, authors)
(new_repo_dir / "pyproject.toml").write_text(pyproject_content)
result = subprocess.run(
["poetry", "lock"],
capture_output=True,
text=True,
check=False,
cwd=new_repo_dir,
)
if result.returncode != 0:
msg = f"Failed to lock packages:\n{result.stderr}"
raise ValueError(msg)
Comment on lines +140 to +154
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. The poetry.lock is now generated during the scaffolding process. Since it’s no longer included as a template (I removed it, it was previously copied over from the templates directory but never had a .template suffix), this slows down the initial setup of an autonomy repo. However:

    • It will always produce a lockfile, assuming adev's own locking is valid (though we can’t simply copy adev's poetry.lock).
    • We could reintroduce a poetry.lock as a template to speed up setup, but the poetry lock step can stay. It’s acceptable for the lockfile to be regenerated if missing; this shouldn’t cause test failures or CI breaks, as it's merely a convenience, not a correctness issue.
  2. For the Python repo scaffolding, I haven’t added a pyproject.jinja (out of scope for this PR). The existing template could use an update, but it’s not urgent for our purposes right now


for file in track(
self.template_files,
description=f"Scaffolding {self.type_of_repo} repo",
Expand Down Expand Up @@ -286,7 +337,7 @@ def scaffold_new_repo(logger, name, type_of_repo, force, auto_approve, install,
src_dir = Path(name)
src_dir.mkdir(exist_ok=False)
logger.debug(f"Scaffolding `{src_dir!s}`")
(src_dir / "__init__.py").touch()
(src_dir / "__init__.py").write_text(f'"""{name.capitalize()} module."""')
(src_dir / "main.py").write_text(SAMPLE_PYTHON_MAIN_FILE)
(src_dir / "cli.py").write_text(SAMPLE_PYTHON_CLI_FILE.format(project_name=name))
else:
Expand Down
2 changes: 2 additions & 0 deletions auto_dev/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
from aea.configurations.data_types import PublicId


THIS_REPO_ROOT = Path(__file__).parent.parent
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is useful because when working with isolated filesystems, functions like get_repo_root correctly point to the newly scaffolded repo. But there are cases where we still need access to the adev repo, e.g., to fetch the current pyproject.toml for generating a new repo. This allows us to reach back into adev even when the test context is scoped to the isolated filesystem.


DEFAULT_ENCODING = "utf-8"
DEFAULT_TZ = "UTC"
DEFAULT_TIMEOUT = 10
Expand Down
4,708 changes: 0 additions & 4,708 deletions auto_dev/data/repo/templates/autonomy/poetry.lock

This file was deleted.

31 changes: 0 additions & 31 deletions auto_dev/data/repo/templates/autonomy/pyproject.toml.template

This file was deleted.

2 changes: 0 additions & 2 deletions auto_dev/data/repo/templates/python/pyproject.toml.template
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ classifiers = [
python = ">=3.10,<3.14"
autonomy-dev = {{extras = ["all"], version = ">=0.2.64,<=0.2.153"}}

[tool.poetry.dev-dependencies]


[tool.poetry.extras]
dev = ["pre-commit", "virtualenv", "pip", "twine", "toml", "bump2version"]
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Tests module"""
24 changes: 24 additions & 0 deletions auto_dev/data/templates/repo/autonomy/pyproject.jinja
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
[build-system]
{{ build_system }}

[tool.poetry]
name = "{{ project_name }}"
version = "0.1.0"
description = ""
authors = ["{{ authors }}"]
readme = "README.md"
license = "Apache-2.0"
classifiers = [
{{ classifiers }}
]
package-mode = false

[tool.poetry.dependencies]
python = "{{ python }}"
autonomy-dev = {extras = ["all"], version = "{{ version }}"}

[tool.poetry.group.dev.dependencies]
{{ dev_dependencies }}

[tool.black]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should ruff formatting be preferred?

{{ black_config }}
3 changes: 1 addition & 2 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,9 @@ def test_filesystem(monkeypatch):


@pytest.fixture
def test_clean_filesystem(monkeypatch):
def test_clean_filesystem():
"""Fixture for invoking command-line interfaces."""
with isolated_filesystem() as directory:
monkeypatch.setenv("PYTHONPATH", directory)
yield directory
Comment on lines 77 to 81
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

test_clean_filesystem is used for testing setting up new repositories. In such case, we can neither know or set the PYTHONPATH a priori



Expand Down
Loading
Loading