Introduce Poetry along with small improvements

.github/workflows/python-app.yml

@@ -16,12 +16,17 @@ jobs:
         python-version:
         - "3.10"
         - "3.11"
+        - "3.12"
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        virtualenvs-create: false
     - name: Pre-install dependencies
       run: |
         make install_lint_requirements
@@ -31,4 +36,4 @@ jobs:
         make lint
     - name: Test
       run: |
-        make test
+        make test PYTEST_OPTS="-vvv"

Makefile

@@ -1,22 +1,36 @@
+POETRY ?= poetry
+_RUN   ?= $(POETRY) run
+PIP    ?= pip3
+PYTEST ?= $(_RUN) pytest
+FLAKE8 ?= $(_RUN) flake8
+BLACK  ?= $(_RUN) black
+PYLINT ?= $(_RUN) pylint
+ISORT  ?= $(_RUN) isort
+MYPY   ?= $(_RUN) mypy
+
+COOKIECUTTER ?= cookiecutter
+
+TESTS_DIR := tests
+
 .PHONY: install_lint_requirements
 install_lint_requirements:
-	pip3 install -e .[lint]
+	$(POETRY) install --with lint
 
 .PHONY: lint
 lint: install_lint_requirements
-	flake8 tests
-	black --line-length=79 --check --diff tests
-	pylint tests
-	isort --check-only tests setup.py
-	mypy tests
+	$(FLAKE8) $(TESTS_DIR)
+	$(BLACK) --check --diff $(TESTS_DIR)
+	$(PYLINT) $(TESTS_DIR)
+	$(ISORT) --check-only $(TESTS_DIR)
+	$(MYPY) $(TESTS_DIR)
 
 .PHONY: install_test_requirements
 install_test_requirements:
-	pip3 install -e .[test]
+	$(POETRY) install --with test
 
 .PHONY: test
 test: install_test_requirements
-	pytest tests
+	$(PYTEST) $(PYTEST_OPTS) $(TESTS_DIR)
 
 .PHONY: clean
 clean:
@@ -29,16 +43,16 @@ clean:
 	find . -regex "*.py[co]" -delete
 
 .PHONY: format
-format:
-	black --line-length=79 tests
-	isort tests setup.py
+format: install_lint_requirements
+	$(BLACK) $(TESTS_DIR)
+	$(ISORT) $(TESTS_DIR)
 
 .PHONY: install
 install:
-	pip3 install .
+	$(PIP) install .
 
-TARGET_DIR := .
+TARGET_DIR ?= .
 
 .PHONY: generate
 generate: install
-	cookiecutter -v . --output-dir="$(TARGET_DIR)"
+	$(COOKIECUTTER) -v . --output-dir="$(TARGET_DIR)"

README.md

@@ -1,15 +1,15 @@
 # python-project-template
 
-![Test status](https://github.com/matyaskuti/python-project-template/actions/workflows/python-app.yml/badge.svg
-)
+![Test status](https://github.com/matyaskuti/python-project-template/actions/workflows/python-app.yml/badge.svg)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
 A [`cookiecutter`](https://github.com/audreyr/cookiecutter) based Python
 project template.
 
 This is an opinionated template, based on useful defaults that we like to have
-when creating new projects. We include a pre-built makefile, with rules for
+when creating new projects. We include a pre-built Makefile, with rules for
 linting and test, scaffolded unit tests, and tools for building wheels,
-amongst other things. 
+amongst other things.
 
 This project is open source because we think it might be useful to other
 engineers. However, Mendix does not officially support this project.
@@ -22,8 +22,8 @@ This project is licensed under the MIT license.
 
 In the below sections it is explained how to generate a new Python package with
 this project. When generating a new package, the tool will request a series of
-inputs, such as the package name, description, author, whether to include
-certain tooling, etc.
+inputs, such as the package name, description, author, tooling such as
+package management, etc.
 
 ### By cloning this repo
 
@@ -41,85 +41,20 @@ be installed automatically._
 2. Run `cookiecutter <repository URL>`
 
 To see what options `cookiecutter` offers (eg. output/target directory,
-verbosit, etc.), run `cookiecutter --help`.
+verbosity, etc.), run `cookiecutter --help`.
 
 ### Remove clutter
 
 In order to be able to test that the package is generated correctly and linting
 and tests can be run, there is a `dummy.py` and a corresponding `test_dummy.py`
 file generated. This is exactly what the name suggests and should be removed.
 
-### Pushing to Git
-
-Make sure you have created a new repository in GitLab/GitHub/etc. already.
-
-After having the desired package generated you can
-* Run `git init` in the new project root and add the existing remote repository
-with `git remote add origin <repository URL>`
-* Or if you have the empty repository already cloned on your machine, copy the
-generated files to the cloned local repository
-* Then all you have to do is push
-
-## Usage - existing project
-
-Since many times we want to improve existing projects instead of generating a
-new one, this tool can also be used to do so, with some extra manual steps
-along the way.
-
-So in case you wish to migrate an existing Python project to comply with this
-template, do the following steps
-
-1. Clone the existing repository
-2. Make sure you are able to use this project on your machine (see the usage
-for a new project above: clone/install cookiecutter)
-3. Generate a new empty project, with the same name as your existing one
-(this is an important step, since later you don't want to manually modify the
-``Makefile`` and ``setup.py`` too much)
-4. From the generated project, move the following files, as-is to your existing
-local repository
-    * ``.gitignore`` (just to be sure, diff it in case your project contains
-    more ignored patterns than the new one)
-    * ``Makefile``
-    * ``pylintrc`` (if applicable)
-    * ``tests`` (if it doesn't exist yet)
-5. Rename the existing ``setup.py`` to ``setup.py.bak``
-6. Move the generated ``setup.py`` to the existing local repository
-7. Merge ``setup.py.bak`` into ``setup.py``
-    * Move entry points
-    * Change description if needed
-    * Adjust the `packages` parameter of the `setup(...)` call if needed,
-    although `find_packages()` should suffice in 99% of cases
-    * Update the `install_requires` parameter with the requirements of the
-    existing package
-    * Create a ``metadata.py`` within the new project's main Python package and
-    make sure the version is correct (`VERSION` and `__version__` parameters)
-    * Make sure you don't lose any extras that are in the setup file, such as
-    extra package data, reference to ``MANIFEST.in``, etc.
-8. Remove ``setup.py.bak``
-9. Remove ``tests/test_dummy.py`` and make there is at least one test to be run
-10. Do a sanity check on the make targets
-    * format
-    * lint
-    * test
-    * build
-    * clean
-11. Make sure tests and linting are green - it could be that making linting
-pass requires a bit of manual work in the code
-    * `flake8`, `pylint`, `black` errors should be easy to fix or explicitly
-    ignore (note that `pylint` errors/warnings that cannot be immediately fixed
-    are usually caused by some deeper design smell in the code, maybe just
-    ignore these at first and come back to fixing them later)
-    * `mypy` can break if some dependencies are not implementing type hinting
-    in this case check out the
-    [documentation](https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports)
-    to explicitly ignore import problems related to this
-12. Remove the newly generated project
-
 ## About the contents of this repository
 
 This project makes use of the following tools (similarly to the generated
 Python package - see below):
 * `make`
+* `poetry`
 * `cookiecutter`
 * `pytest`
 * `pytest-cookies`
@@ -140,9 +75,15 @@ correct
 values of project parameters
 
 In order to easily test proper generation of a Python project, a `pytest`
-plugin, `pytest-cookies` is used. This provides a `cookies` fixture, which is
-injected into the test cases during runtime, making it really easy to test-run
-the `cookiecutter` template in an auto-generated location.
+plugin, [`pytest-cookies`](https://github.com/hackebrot/pytest-cookies) is
+used. This provides a `cookies` fixture, which is injected into the test cases
+during runtime, making it really easy to test-run the `cookiecutter` template
+in an auto-generated location.
+
+While the project uses [Poetry](https://python-poetry.org/) as a package
+manager, to install and use it (ie. to run `make generate`), does not require
+Poetry, only Pip (which is assumed to be part of most standard Python
+installations). Only contributing requires Poetry.
 
 ## About the generated Python project
 
@@ -159,10 +100,10 @@ Below are the main `make` targets and the tools used within:
     still leaves a lot of flexibility and there are as many preferences as
     developers, we use this tool because it is already opinionated so you don't
     have to be
+    * `isort` - linter and formatter specialized for imports
     * `pylint` - linting, error and duplication detection and very much
-    customizable; the generated project contains a minimal, but decent
-    `pylintrc` configuration file; its usage is optional, can be decided upon
-    project generation, however highly recommended and turned on by default
+    customizable; the generated project contains a minimal, but decent set of
+    configuration
     * `mypy` - type checker, the de facto standard at the moment
 * `format` - to easily comply with the above standards at the push of a button
     * `black` - because of the reasons mentioned above
@@ -177,33 +118,21 @@ reports, etc.
 * `build` - to create a standard, distributable Python package
     * `wheel` - this is the current standard for creating distributables
 
-_Note: the targets `lint`, `test` and `build` have a corresponding
+_Note: the targets `lint` and `test` have a corresponding
 `install_<target>_requirements` target to install extra dependencies. These are
-individually defined in the generated project's `setup.py` as well as extra
-requirements. There is no need to call the install targets on their own, they
-are called automatically in their related main target._
-
-### Future extension
-
-New linters can be easily added by extending the `Makefile`, potentially made
-optional (just as with `pylint`).
-
-Currently in the created project there is only one `test` target which is
-intendet to be used to run a set of automated tests in the "commit phase".
-However eventually there should be more testing targets created, thus
-separating different levels of automated tests, such as
-* Integration tests (`test-integration`) - automatically verifying the
-application is piped correctly to other system components
-* Acceptance tests (`test-acceptance` target) - automatically verifying
-functional and non-functional requirements, potentially in a BDD style
-* Capacity tests (`test-capacity` target) - automatically verifying that an
-application is able to handle load according to requirements
-* Security (`security` target), to run some automated security tooling
-(eg. Snyk or BlackDuck) to reveal potential vurnelabilities in the application
-code itself or introduced by dependencies
-
-In addition to this we could introduce automated documentation generation in
-the created project, using [Sphinx](http://www.sphinx-doc.org/en/master/) via
-a `make docs` target. For this we will need some storage to be able to host the
-generated docs and push to it from Python projects upon a successful master
-build.
+individually defined in the generated project's `pyproject.toml` as well, as
+extra requirements. There is no need to call the install targets on their own,
+they are called automatically in their related main target._
+
+### Dependency and package management
+
+A single ``pyproject.toml`` file is used for the generated project's
+definition, packaging and tooling configuration.
+When generating the project, the `build_system` parameter decides whether the
+created Python package use [Poetry](https://python-poetry.org/) or
+[Setuptools](https://setuptools.pypa.io/) for dependency management and as a
+build backend: it defaults to Poetry, if any other value is provided then
+Setuptools will be used.
+
+Picking either will be reflected in the ``pyproject.toml`` and the
+``Makefile``.

cookiecutter.json

@@ -3,5 +3,6 @@
     "short_description": "Python project",
     "author_name": "Mendix Cloud Value Added Services Team",
     "author_email": "dis_valueaddedservices@mendix.com",
-    "use_pylint": "y"
+    "build_system": "poetry",
+    "line_length": 79
 }

hooks/post_gen_project.py

@@ -1,8 +1,13 @@
 import os
+import subprocess
 
 PROJECT_DIRECTORY = os.path.realpath(os.path.curdir)
 
 
 if __name__ == "__main__":
-    if "{{ cookiecutter.use_pylint }}" != "y":
-        os.remove(os.path.join(PROJECT_DIRECTORY, "pylintrc"))
+    # if " cookiecutter.use_sometool }}" != "y":
+    #     os.remove(os.path.join(PROJECT_DIRECTORY, "sometoolrc"))
+    # The above couple lines are left as an example for the future.
+
+    if "{{ cookiecutter.build_system }}" == "poetry":
+        subprocess.call(["poetry", "lock"], stderr=subprocess.STDOUT)

hooks/pre_gen_project.py

@@ -4,6 +4,11 @@
 MODULE_REGEX = r"^[_a-zA-Z][_a-zA-Z0-9]+$"
 PACKAGE_NAME = "{{ cookiecutter.package_name }}"
 
+LINE_LENGTH = int("{{ cookiecutter.line_length }}")
+
 if __name__ == "__main__":
     if not re.match(MODULE_REGEX, PACKAGE_NAME):
         sys.exit("ERROR: The package name is not a valid Python module name.")
+
+    if LINE_LENGTH < 79:
+        sys.exit("ERROR: The line length must be at least 79.")