diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a7da552b..c3b9d5e3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,6 +54,13 @@ jobs: - name: Format with Ruff run: ruff format --check . + - name: Lint with Flake8 (for docstrings) + if: ${{ contains('3.12,3.13', matrix.python) }} + # Flake8 crashes on Python < 3.12, so we exclude those versions. + # Flake8 is primarily here to lint the docstrings, which + # does not need to happen under multiple versions. + run: flake8 src + - name: Test with pytest run: pytest --cov=src --cov-report=lcov diff --git a/.readthedocs.yaml b/.readthedocs.yaml index d016071b..368064de 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,7 +8,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.11" + python: "3.12" # Build documentation in the "docs/" directory with Sphinx sphinx: @@ -24,4 +24,5 @@ sphinx: # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html python: install: - - requirements: docs/requirements.txt + - requirements: dev-requirements.txt + - path: . diff --git a/README.md b/README.md index 58b56399..c5faabf5 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,9 @@ # labthings-fastapi -A FastAPI based library to implement a [Web of Things] interface for laboratory hardware using Python. This is a ground-up rewrite of [python-labthings], replacing Flask 1 and Marshmallow with FastAPI and Pydantic. It is the underlying framework for v3 of the [OpenFlexure Microscope software]. +A FastAPI based library to implement a [Web of Things] interface for laboratory hardware using Python. This is a ground-up rewrite of [python-labthings], based on FastAPI and Pydantic. It is the underlying framework for v3 of the [OpenFlexure Microscope software]. + +Documentation, including install instructions, is available on [readthedocs]. Features include: @@ -18,7 +20,7 @@ Features include: - Dependency injection is used to manage relationships between Things and dependency on the server * Async HTTP handling - Starlette (used by FastAPI) can handle requests asynchronously - potential for websockets/events (not used much yet) - - `Thing` code is still, for now, threaded. I intend to make it possible to write async things in the future, but don't intend it to become mandatory + - `Thing` code is still, for now, threaded. It may become possible to write async things in the future, but won't become mandatory * Smaller codebase - FastAPI more or less completely eliminates OpenAPI generation code from our codebase - Thing Description generation is very much simplified by the new structure (multiple Things instead of one massive Thing with many extensions) @@ -28,11 +30,11 @@ Features include: ## Installation -You can install this repository with `pip install labthings-fastapi`. It may at some point be renamed to `labthings` v2. For the latest development version, either clone it and run `pip install -e .[dev]` to work on it, or just `pip install https://gitlab.com/rwb27/labthings-fastapi.git`. +See [readthedocs] for installation instructions that are automatically tested. You can install this package with `pip install labthings-fastapi`. It may at some point be renamed to `labthings` v2. For the latest development version, either clone it and run `pip install -e .[dev]` to work on it, or just `pip install https://gitlab.com/rwb27/labthings-fastapi.git`. ## Developer notes -The code is linted with `ruff .`, type checked with `mypy src`, and tested with `pytest`. These all run in CI with GitHub Actions. The codebase is not even `v0.1` yet so it's still subject to summary rearrangement. We recommend a [pre-commit hook] to ensure `ruff` passes on every commit. +The code is linted with `ruff .`, type checked with `mypy src`, and tested with `pytest`. These all run in CI with GitHub Actions. The codebase is not even `v0.1` yet so it's still subject to summary rearrangement. We recommend a [pre-commit hook] to ensure `ruff` passes on every commit. `flake8` is also run in CI, primarily to enable stricter checks on docstrings. It is run as `flake8 src`. `ruff` and `flake8` are both configured from `pyproject.toml`. Dependencies are defined in `pyproject.toml` and can be compiled to `dev-requirements.txt` with: ``` @@ -44,9 +46,10 @@ All changes to the codebase should go via pull requests, and should only be merg ## Demo -See the [examples folder](./examples/) for a runnable demo. +See [readthedocs] for a runnable demo. [Web of Things]: https://www.w3.org/WoT/ [python-labthings]: https://github.com/labthings/python-labthings/ [OpenFlexure Microscope software]: https://gitlab.com/openflexure/openflexure-microscope-server/ [pre-commit hook]: https://openflexure.org/contribute#use-git-hooks-for-ci-checks +[readthedocs]: https://labthings-fastapi.readthedocs.io/ diff --git a/dev-requirements.txt b/dev-requirements.txt index be256006..2982bb8b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,23 +1,34 @@ # This file was autogenerated by uv via the following command: # uv pip compile --extra dev pyproject.toml --output-file dev-requirements.txt +alabaster==1.0.0 + # via sphinx annotated-types==0.7.0 # via pydantic -anyio==4.8.0 +anyio==4.9.0 # via # labthings-fastapi (pyproject.toml) # httpx # starlette # watchfiles -attrs==25.1.0 +astroid==3.3.11 + # via sphinx-autoapi +attrs==25.3.0 # via # jsonschema # referencing -certifi==2025.1.31 +babel==2.17.0 + # via sphinx +certifi==2025.7.14 # via # httpcore # httpx -click==8.1.8 + # requests + # sentry-sdk +charset-normalizer==3.4.2 + # via requests +click==8.2.1 # via + # pydoclint # rich-toolkit # typer # uvicorn @@ -25,22 +36,54 @@ colorama==0.4.6 # via # click # pytest + # sphinx # uvicorn -coverage==7.6.12 +coverage==7.9.2 # via pytest-cov dnspython==2.7.0 # via email-validator +docstring-parser-fork==0.0.12 + # via pydoclint +docutils==0.21.2 + # via + # restructuredtext-lint + # sphinx + # sphinx-rtd-theme email-validator==2.2.0 - # via fastapi -fastapi==0.115.11 + # via + # fastapi + # pydantic +exceptiongroup==1.3.0 + # via + # anyio + # pytest +fastapi==0.116.1 # via labthings-fastapi (pyproject.toml) -fastapi-cli==0.0.7 +fastapi-cli==0.0.8 # via fastapi -h11==0.14.0 +fastapi-cloud-cli==0.1.4 + # via fastapi-cli +flake8==7.3.0 + # via + # labthings-fastapi (pyproject.toml) + # flake8-docstrings + # flake8-pyproject + # flake8-rst + # flake8-rst-docstrings + # pydoclint +flake8-docstrings==1.7.0 + # via labthings-fastapi (pyproject.toml) +flake8-pyproject==1.2.3 + # via labthings-fastapi (pyproject.toml) +flake8-rst==0.8.0 + # via labthings-fastapi (pyproject.toml) +flake8-rst-docstrings==0.3.1 + # via labthings-fastapi (pyproject.toml) +h11==0.16.0 # via # httpcore # uvicorn -httpcore==1.0.7 +httpcore==1.0.9 # via httpx httptools==0.6.4 # via uvicorn @@ -48,64 +91,91 @@ httpx==0.28.1 # via # labthings-fastapi (pyproject.toml) # fastapi + # fastapi-cloud-cli idna==3.10 # via # anyio # email-validator # httpx + # requests ifaddr==0.2.0 # via zeroconf -iniconfig==2.0.0 +imagesize==1.4.1 + # via sphinx +iniconfig==2.1.0 # via pytest itsdangerous==2.2.0 # via fastapi jinja2==3.1.6 - # via fastapi -jsonschema==4.23.0 + # via + # fastapi + # sphinx + # sphinx-autoapi +jsonschema==4.24.1 # via labthings-fastapi (pyproject.toml) -jsonschema-specifications==2024.10.1 +jsonschema-specifications==2025.4.1 # via jsonschema markdown-it-py==3.0.0 # via rich markupsafe==3.0.2 # via jinja2 +mccabe==0.7.0 + # via flake8 mdurl==0.1.2 # via markdown-it-py -mypy==1.15.0 +mypy==1.17.0 # via labthings-fastapi (pyproject.toml) -mypy-extensions==1.0.0 +mypy-extensions==1.1.0 # via mypy -numpy==2.2.3 +numpy==2.2.6 # via labthings-fastapi (pyproject.toml) -orjson==3.10.15 +orjson==3.11.0 # via fastapi -packaging==24.2 - # via pytest +packaging==25.0 + # via + # pytest + # sphinx +pathspec==0.12.1 + # via mypy pillow==11.3.0 # via labthings-fastapi (pyproject.toml) -pluggy==1.5.0 - # via pytest +pluggy==1.6.0 + # via + # pytest + # pytest-cov +pycodestyle==2.14.0 + # via flake8 pydantic==2.10.6 # via # labthings-fastapi (pyproject.toml) # fastapi + # fastapi-cloud-cli # pydantic-extra-types # pydantic-settings pydantic-core==2.27.2 # via pydantic -pydantic-extra-types==2.10.2 +pydantic-extra-types==2.10.5 # via fastapi -pydantic-settings==2.8.1 +pydantic-settings==2.10.1 # via fastapi -pygments==2.19.1 - # via rich +pydoclint==0.6.6 + # via labthings-fastapi (pyproject.toml) +pydocstyle==6.3.0 + # via flake8-docstrings +pyflakes==3.4.0 + # via flake8 +pygments==2.19.2 + # via + # flake8-rst-docstrings + # rich + # sphinx pytest==7.4.4 # via # labthings-fastapi (pyproject.toml) # pytest-cov -pytest-cov==6.0.0 +pytest-cov==6.2.1 # via labthings-fastapi (pyproject.toml) -python-dotenv==1.0.1 +python-dotenv==1.1.1 # via # pydantic-settings # uvicorn @@ -114,55 +184,117 @@ python-multipart==0.0.20 pyyaml==6.0.2 # via # fastapi + # sphinx-autoapi # uvicorn referencing==0.36.2 # via # jsonschema # jsonschema-specifications # types-jsonschema -rich==13.9.4 +requests==2.32.4 + # via sphinx +restructuredtext-lint==1.4.0 + # via flake8-rst-docstrings +rich==14.0.0 # via # rich-toolkit # typer -rich-toolkit==0.13.2 - # via fastapi-cli -rpds-py==0.23.1 +rich-toolkit==0.14.8 + # via + # fastapi-cli + # fastapi-cloud-cli +rignore==0.6.2 + # via fastapi-cloud-cli +rpds-py==0.26.0 # via # jsonschema # referencing -ruff==0.9.10 +ruff==0.12.3 # via labthings-fastapi (pyproject.toml) +sentry-sdk==2.33.0 + # via fastapi-cloud-cli shellingham==1.5.4 # via typer sniffio==1.3.1 # via anyio -starlette==0.46.0 +snowballstemmer==3.0.1 + # via + # pydocstyle + # sphinx +sphinx==8.1.3 + # via + # labthings-fastapi (pyproject.toml) + # sphinx-autoapi + # sphinx-rtd-theme + # sphinxcontrib-jquery +sphinx-autoapi==3.6.0 + # via labthings-fastapi (pyproject.toml) +sphinx-rtd-theme==3.0.2 + # via labthings-fastapi (pyproject.toml) +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +starlette==0.47.1 # via fastapi -typer==0.15.2 - # via fastapi-cli -types-jsonschema==4.23.0.20241208 +tomli==2.2.1 + # via + # coverage + # flake8-pyproject + # mypy + # pydoclint + # pytest + # sphinx +typer==0.16.0 + # via + # fastapi-cli + # fastapi-cloud-cli +types-jsonschema==4.24.0.20250708 # via labthings-fastapi (pyproject.toml) -typing-extensions==4.12.2 +typing-extensions==4.14.1 # via # labthings-fastapi (pyproject.toml) # anyio + # astroid + # exceptiongroup # fastapi # mypy # pydantic # pydantic-core # pydantic-extra-types # referencing + # rich # rich-toolkit + # starlette # typer + # typing-inspection + # uvicorn +typing-inspection==0.4.1 + # via pydantic-settings ujson==5.10.0 # via fastapi -uvicorn==0.34.0 +urllib3==2.5.0 + # via + # requests + # sentry-sdk +uvicorn==0.35.0 # via # fastapi # fastapi-cli -watchfiles==1.0.4 + # fastapi-cloud-cli +watchfiles==1.1.0 # via uvicorn websockets==15.0.1 # via uvicorn -zeroconf==0.146.1 +zeroconf==0.147.0 # via labthings-fastapi (pyproject.toml) diff --git a/docs/requirements.in b/docs/requirements.in deleted file mode 100644 index 8d2d98c1..00000000 --- a/docs/requirements.in +++ /dev/null @@ -1,4 +0,0 @@ -sphinx-autodoc2==0.5.0 -mdit-py-plugins>=0.3.4 -myst-parser>=3.0.1, <4 -sphinx-rtd-theme==2.0.0 diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index e88a31da..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,32 +0,0 @@ -alabaster==0.7.16 -astroid==3.2.1 -Babel==2.15.0 -certifi==2024.2.2 -charset-normalizer==3.3.2 -colorama==0.4.6 -docutils==0.20.1 -idna==3.7 -imagesize==1.4.1 -Jinja2==3.1.4 -markdown-it-py==3.0.0 -MarkupSafe==2.1.5 -mdit-py-plugins==0.4.1 -mdurl==0.1.2 -myst-parser==3.0.1 -packaging==24.0 -Pygments==2.18.0 -PyYAML==6.0.1 -requests==2.31.0 -snowballstemmer==2.2.0 -Sphinx==7.3.7 -sphinx-autodoc2==0.5.0 -sphinx-rtd-theme==2.0.0 -sphinxcontrib-applehelp==1.0.8 -sphinxcontrib-devhelp==1.0.6 -sphinxcontrib-htmlhelp==2.0.5 -sphinxcontrib-jquery==4.1 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.7 -sphinxcontrib-serializinghtml==1.1.10 -typing_extensions==4.11.0 -urllib3==2.2.1 \ No newline at end of file diff --git a/docs/source/.gitignore b/docs/source/.gitignore index df77b72e..d91ca728 100644 --- a/docs/source/.gitignore +++ b/docs/source/.gitignore @@ -1,3 +1,4 @@ /build/ .venv/ /apidocs/ +/autoapi/ \ No newline at end of file diff --git a/docs/source/actions.rst b/docs/source/actions.rst new file mode 100644 index 00000000..b0f834bd --- /dev/null +++ b/docs/source/actions.rst @@ -0,0 +1,59 @@ +.. _actions: + +Actions +======= + +Actions are the way `.Thing` objects are instructed to do things. In Python +terms, any method of a `.Thing` that we want to be able to call over HTTP +should be decorated as an Action, using :deco:`.thing_action`. + +This page gives an overview of how actions are implemented in LabThings-FastAPI. +:ref:`wot_cc` includes a section on :ref:`wot_actions` that introduces the general concept. + +Running actions via HTTP +------------------------ + +LabThings-FastAPI allows these methods to be invoked over HTTP, and +each invocation runs in its own thread. Currently, the ``POST`` request that +invokes an action will return almost immediately with a ``201`` code, and a +JSON payload that describes the invocation as an `.InvocationModel`. This includes +a link ``href`` that can be polled to check the status of the invocation. + +The HTTP implementation of `.ThingClient` first makes a ``POST`` request to +invoke the action, then polls the invocation using the ``href`` supplied. +Once the action has finished (i.e. its status is ``completed``, ``error``, or +``cancelled``), its output (the return value) is retrieved and used as the +return value. + +On the server, when an action is invoked over HTTP, we create a new +`.Invocation`, which is a subclass of `threading.Thread`, to run it in parallel +with other code, and keep track of its progress. The log output and return value +are held by the `.Invocation` object. + +Actions are supported in LabThings-FastAPI by an `.ActionManager`, responsible +for keeping track of all the running and recently-completed Actions. This is +where Invocation-related HTTP endpoints are handled, including listing all the +`.Invocation` objects and returning the status of an individual `.Invocation`. + +Running actions from other actions +---------------------------------- + +If code running in a `.Thing` runs methods belonging either to that `.Thing` +or to another `.Thing` on the same server, no new thread is created: the +called action runs in the same thread as the calling action, just like any +other Python code. + +Action inputs and outputs +------------------------- +The code that implements an action is a method of a `.Thing`, meaning it is +a function. The input parameters are the function's arguments, and the output +parameter is the function's return value. Type hints on both arguments and +return value are used to document the action in the OpenAPI description and +the Thing Description, so it is important to use them consistently. + +There are some function arguments that are not considered input parameters. +The first is ``self`` (the first positional argument), which is always the +`.Thing` on which the argument is defined. The other special arguments are +:ref:`dependencies`, which use annotated type hints to tell LabThings to +supply resources needed by the action. Most often, this is a way of accessing +other `.Things` on the same server. diff --git a/docs/source/blobs.rst b/docs/source/blobs.rst index c39e82bf..ebdba5cf 100644 --- a/docs/source/blobs.rst +++ b/docs/source/blobs.rst @@ -1,31 +1,33 @@ +.. _blobs: + Blob input/output ================= -:class:`.Blob` objects allow binary data to be returned by an Action. This binary data can be passed between Things, or between Things and client code. Using a :class:`.Blob` object allows binary data to be efficiently sent over HTTP if required, and allows the same code to run either on the server (without copying the data) or on a client (where data is transferred over HTTP). +`.Blob` objects allow binary data to be returned by an Action. This binary data can be passed between Things, or between Things and client code. Using a `.Blob` object allows binary data to be efficiently sent over HTTP if required, and allows the same code to run either on the server (without copying the data) or on a client (where data is transferred over HTTP). -If interactions require only simple data types that can easily be represented in JSON, very little thought needs to be given to data types - strings and numbers will be converted to and from JSON automatically, and your Python code should only ever see native Python datatypes whether it's running on the server or a remote client. However, if you want to transfer larger data objects such as images, large arrays or other binary data, you will need to use a :class:`.Blob` object. +If interactions require only simple data types that can easily be represented in JSON, very little thought needs to be given to data types - strings and numbers will be converted to and from JSON automatically, and your Python code should only ever see native Python datatypes whether it's running on the server or a remote client. However, if you want to transfer larger data objects such as images, large arrays or other binary data, you will need to use a `.Blob` object. -:class:`.Blob` objects are not part of the Web of Things specification, which is most often used with fairly simple data structures in JSON. In LabThings-FastAPI, the :class:`.Blob` mechanism is intended to provide an efficient way to work with arbitrary binary data. If it's used to transfer data between two Things on the same server, the data should not be copied or otherwise iterated over - and when it must be transferred over the network it can be done using a binary transfer, rather than embedding in JSON with base64 encoding. +`.Blob` objects are not part of the Web of Things specification, which doesn't give much consideration to returning large or complicated datatypes. In LabThings-FastAPI, the `.Blob` mechanism is intended to provide an efficient way to work with arbitrary binary data. If it's used to transfer data between two Things on the same server, the data should not be copied or otherwise iterated over - and when it must be transferred over the network it can be done using a binary transfer, rather than embedding in JSON with base64 encoding. -A :class:`.Blob` consists of some data and a MIME type, which sets how the data should be interpreted. It is best to create a subclass of :class:`.Blob` with the content type set: this makes it clear what kind of data is in the :class:`.Blob`. In the future, it might be possible to add functionality to :class:`.Blob` subclasses, for example to make it simple to obtain an image object from a :class:`.Blob` containing JPEG data. However, this will not currently work across both client and server code. +A `.Blob` consists of some data and a MIME type, which sets how the data should be interpreted. It is best to create a subclass of `.Blob` with the content type set: this makes it clear what kind of data is in the `.Blob`. In the future, it might be possible to add functionality to `.Blob` subclasses, for example to make it simple to obtain an image object from a `.Blob` containing JPEG data. However, this will not currently work across both client and server code. -Creating and using :class:`.Blob` objects +Creating and using `.Blob` objects ------------------------------------------------ -Blobs can be created from binary data that is in memory (a :class:`bytes` object), on disk (a file), or using a URL as a placeholder. The intention is that the code that uses a :class:`.Blob` should not need to know which of these is the case, and should be able to use the same code regardless of how the data is stored. +Blobs can be created from binary data that is in memory (a `bytes` object) with `.Blob.from_bytes`, on disk (with `.Blob.from_temporary_directory` or `.Blob.from_file`), or using a URL as a placeholder. The intention is that the code that uses a `.Blob` should not need to know which of these is the case, and should be able to use the same code regardless of how the data is stored. Blobs offer three ways to access their data: -* A `bytes` object, obtained via the `data` property. For blobs created with a `bytes` object, this simply returns the original data object with no copying. If the data is stored in a file, the file is opened and read when the `data` property is accessed. If the :class:`.Blob` references a URL, it is retrieved and returned when `data` is accessed. -* An `open()` method providing a file-like object. This returns a :class:`~io.BytesIO` wrapper if the :class:`.Blob` was created from a `bytes` object or the file if the data is stored on disk. URLs are retrieved, stored as `bytes` and returned wrapped in a :class:`~io.BytesIO` object. -* A `save` method will either save the data to a file, or copy the existing file on disk. This should be more efficient than loading `data` and writing to a file, if the :class:`.Blob` is pointing to a file rather than data in memory. +* A `bytes` object, obtained via the `.Blob.data` property. For blobs created with a `bytes` object, this simply returns the original data object with no copying. If the data is stored in a file, the file is opened and read when the `.Blob.data` property is accessed. If the `.Blob` references a URL, it is retrieved and returned when `.Blob.data` is accessed. +* An `.Blob.open` method providing a file-like object. This returns a `~io.BytesIO` wrapper if the `.Blob` was created from a `bytes` object or the file if the data is stored on disk. URLs are retrieved, stored as `bytes` and returned wrapped in a :class:`~io.BytesIO` object. +* A `.Blob.save` method will either save the data to a file, or copy the existing file on disk. This should be more efficient than loading `.Blob.data` and writing to a file, if the `.Blob` is pointing to a file rather than data in memory. -The intention here is that :class:`.Blob` objects may be used identically with data in memory or on disk or even at a remote URL, and the code that uses them should not need to know which is the case. +The intention here is that `.Blob` objects may be used identically with data in memory or on disk or even at a remote URL, and the code that uses them should not need to know which is the case. Examples -------- -A camera might want to return an image as a :class:`.Blob` object. The code for the action might look like this: +A camera might want to return an image as a `.Blob` object. The code for the action might look like this: .. code-block:: python @@ -57,6 +59,13 @@ The corresponding client code might look like this: img = Image.open(f) img.show() # This will display the image in a window +Using `.Blob` objects as inputs +-------------------------------------- + +`.Blob` objects may be used as either the input or output of an action. There are relatively few good use cases for `.Blob` inputs to actions, but a possible example would be image capture: one action could perform a quick capture of raw data, and another action could convert the raw data into a useful image. The output of the capture action would be a `.Blob` representing the raw data, which could be passed to the conversion action. + +Because `.Blob` outputs are represented in JSON as links, they are downloaded with a separate HTTP request if needed. There is currently no way to create a `.Blob` on the server via HTTP, which means remote clients can use `.Blob` objects provided in the output of actions but they cannot yet upload data to be used as input. However, it is possible to pass the URL of a `.Blob` that already exists on the server as input to a subsequent Action. This means, in the example above of raw image capture, a remote client over HTTP can pass the raw `.Blob` to the conversion action, and the raw data need never be sent over the network. + We could define a more sophisticated camera that can capture raw images and convert them to JPEG, using two actions: .. code-block:: python @@ -110,38 +119,33 @@ On the client, we can use the `capture_image` action directly (as before), or we raw_blob.save("raw_image.raw") # Download and save the raw image to a file - -Using :class:`.Blob` objects as inputs --------------------------------------- - -:class:`.Blob` objects may be used as either the input or output of an action. There are relatively few good use cases for :class:`.Blob` inputs to actions, but a possible example would be image capture: one action could perform a quick capture of raw data, and another action could convert the raw data into a useful image. The output of the capture action would be a :class:`.Blob` representing the raw data, which could be passed to the conversion action. - -Because :class:`.Blob` outputs are represented in JSON as links, they are downloaded with a separate HTTP request if needed. There is currently no way to create a :class:`.Blob` on the server via HTTP, which means remote clients can use :class:`.Blob` objects provided in the output of actions but they cannot yet upload data to be used as input. However, it is possible to pass the URL of a :class:`.Blob` that already exists on the server as input to a subsequent Action. This means, in the example above of raw image capture, a remote client over HTTP can pass the raw :class:`.Blob` to the conversion action, and the raw data need never be sent over the network. - - HTTP interface and serialization -------------------------------- -:class:`.Blob` objects are subclasses of `pydantic.BaseModel`, which means they can be serialized to JSON and deserialized from JSON. When this happens, the :class:`.Blob` is represented as a JSON object with `.Blob.url` and `.Blob.content_type` fields. The `.Blob.url` field is a link to the data. The `.Blob.content_type` field is a string representing the MIME type of the data. It is worth noting that models may be nested: this means an action may return many :class:`.Blob` objects in its output, either as a list or as fields in a :class:`pydantic.BaseModel` subclass. Each :class:`.Blob` in the output will be serialized to JSON with its URL and content type, and the client can then download the data from the URL, one download per :class:`.Blob` object. +`.Blob` objects are subclasses of `pydantic.BaseModel`, which means they can be serialized to JSON and deserialized from JSON. When this happens, the `.Blob` is represented as a JSON object with `.Blob.url` and `.Blob.content_type` fields. The `.Blob.url` field is a link to the data. The `.Blob.content_type` field is a string representing the MIME type of the data. It is worth noting that models may be nested: this means an action may return many `.Blob` objects in its output, either as a list or as fields in a `pydantic.BaseModel` subclass. Each `.Blob` in the output will be serialized to JSON with its URL and content type, and the client can then download the data from the URL, one download per `.Blob` object. -When a :class:`.Blob` is serialized, the URL is generated with a unique ID to allow it to be downloaded. The URL is not guaranteed to be permanent, and should not be used as a long-term reference to the data. The URL will expire after 5 minutes, and the data will no longer be available for download after that time. +When a `.Blob` is serialized, the URL is generated with a unique ID to allow it to be downloaded. The URL is not guaranteed to be permanent, and should not be used as a long-term reference to the data. For `.Blob` objects that are part of the output of an action, the URL will expire after 5 minutes (or the retention time set for the action), and the data will no longer be available for download after that time. In order to run an action and download the data, currently an HTTP client must: -* Call the action that returns a :class:`.Blob` object, which will return a JSON object representing the invocation. -* Poll the invocation until it is complete, and the :class:`.Blob` is available in its ``output`` property with the URL and content type. -* Download the data from the URL in the :class:`.Blob` object, which will return the binary data. +* Call the action that returns a `.Blob` object, which will return a JSON object representing the invocation. +* Poll the invocation until it is complete, and the `.Blob` is available in its ``output`` property with the URL and content type. +* Download the data from the URL in the `.Blob` object, which will return the binary data. It may be possible to have actions return binary data directly in the future, but this is not yet implemented. +.. note:: + + Serialising or deserialising `.Blob` objects requires access to the `.BlobDataManager` associated with the `.ThingServer`. As there is no way to pass this in to the relevant methods at serialisation/deserialisation time, we use context variables to access them. This means that a `.blob_serialisation_context_manager` should be used to set (and then clear) those context variables. This is done by the `.BlobIOContextDep` dependency on the relevant endpoints (currently any endpoint that may return the output of an action). + Memory management and retention ------------------------------- -Management of :class:`.Blob` objects is currently very basic: when a :class:`.Blob` object is returned in the output of an Action that has been called via the HTTP interface, a fixed 5 minute expiry is used. This should be improved in the future to avoid memory management issues. +Management of `.Blob` objects is currently very basic: when a `.Blob` object is returned in the output of an Action that has been called via the HTTP interface, it will be retained as long as the action's output. This may be set on each action, and defaults to 5 minutes. This should be improved in the future to avoid memory management issues. -When a :class:`.Blob` is serialized, a URL is generated with a unique ID to allow it to be downloaded. However, only a weak reference is held to the :class:`.Blob`. Once an Action has finished running, the only strong reference to the :class:`.Blob` should be held by the output property of the action invocation. The :class:`.Blob` should be garbage collected once the output is no longer required, i.e. when the invocation is discarded - currently 5 minutes after the action completes, once the maximum number of invocations has been reached or when it is explicitly deleted by the client. +When a `.Blob` is serialized, a URL is generated with a unique ID to allow it to be downloaded. However, only a weak reference is held to the `.Blob`. Once an Action has finished running, the only strong reference to the `.Blob` should be held by the output property of the action invocation. The `.Blob` should be garbage collected once the output is no longer required, i.e. when the invocation is discarded - currently 5 minutes after the action completes, once the maximum number of invocations has been reached or when it is explicitly deleted by the client. -The behaviour is different when actions are called from other actions. If `action_a` calls `action_b`, and `action_b` returns a :class:`.Blob`, that :class:`.Blob` will be subject to Python's usual garbage collection rules when `action_a` ends - i.e. it will not be retained unless it is included in the output of `action_a`. +The behaviour is different when actions are called from other actions. If `action_a` calls `action_b`, and `action_b` returns a `.Blob`, that `.Blob` will be subject to Python's usual garbage collection rules when `action_a` ends - i.e. it will not be retained unless it is included in the output of `action_a`. diff --git a/docs/source/concurrency.rst b/docs/source/concurrency.rst index 3eaad64e..3e04e246 100644 --- a/docs/source/concurrency.rst +++ b/docs/source/concurrency.rst @@ -1,3 +1,5 @@ +.. _concurrency: + Concurrency in LabThings-FastAPI ================================== diff --git a/docs/source/conf.py b/docs/source/conf.py index 551ae884..3acee8d0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,3 +1,6 @@ +import logging +import labthings_fastapi + # Configuration file for the Sphinx documentation builder. # # For the full list of built-in configuration values, see the documentation: @@ -9,16 +12,16 @@ project = "labthings-fastapi" copyright = "2024, Richard Bowman" author = "Richard Bowman" -release = "0.0.1" +release = "0.0.10" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - "myst_parser", "sphinx.ext.intersphinx", # "sphinx.ext.napoleon", - "autodoc2", + # "autodoc2", + "autoapi.extension", "sphinx_rtd_theme", ] @@ -27,14 +30,14 @@ default_role = "py:obj" -autodoc2_packages = ["../../src/labthings_fastapi"] -autodoc2_render_plugin = "myst" -autodoc2_class_docstring = "both" +# autodoc2_packages = ["../../src/labthings_fastapi"] +# autodoc2_render_plugin = "myst" +# autodoc2_class_docstring = "both" -# autoapi_dirs = ["../../src/labthings_fastapi"] -# autoapi_ignore = [] -# autoapi_generate_api_docs = True -# autoapi_keep_files = True +autoapi_dirs = ["../../src/labthings_fastapi"] +autoapi_generate_api_docs = True +autoapi_keep_files = True +autoapi_python_class_content = "both" # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output @@ -47,6 +50,62 @@ "fastapi": ("https://fastapi.tiangolo.com", None), "anyio": ("https://anyio.readthedocs.io/en/stable/", None), "pydantic": ("https://docs.pydantic.dev/latest/", None), + "jsonschema": ("https://python-jsonschema.readthedocs.io/en/stable/", None), + "numpy": ("https://numpy.org/doc/stable/", None), } -myst_enable_extensions = ["fieldlist"] +# The next section deals with skipping names. Because various modules import +# symbols with `from x import y`, those symbols are duplicated by apidoc. +# The logic below defines a function that skips functions we've pulled into +# the public API, and functions that are used elsewhere, to ensure they +# are documented exactly once, at the fully qualified name specified. + +skipper_log = logging.getLogger("skipper") +skipper_log.addHandler(logging.FileHandler("./skipper.log", mode="w")) +skipper_log.setLevel(logging.DEBUG) + +convenience_modules = { + "labthings_fastapi": labthings_fastapi.__all__, + "labthings_fastapi.deps": labthings_fastapi.deps.__all__, +} +canonical_fq_names = [ + "labthings_fastapi.descriptors.action.ActionDescriptor", + "labthings_fastapi.outputs.blob.BlobDataManager", + "labthings_fastapi.actions.invocation_model.InvocationModel", + "labthings_fastapi.outputs.MJPEGStream", + "labthings_fastapi.outputs.MJPEGStreamDescriptor", + "labthings_fastapi.outputs.blob.BlobIOContextDep", + "labthings_fastapi.actions.ActionManager", + "labthings_fastapi.descriptors.endpoint.EndpointDescriptor", + "labthings_fastapi.dependencies.invocation.invocation_logger", + "labthings_fastapi.utilities.introspection.EmptyObject", +] + + +def unqual(name): + if "." in name: + return name.split(".")[-1] + return name + + +canonical_names = {unqual(n): n for n in canonical_fq_names} + +skipper_log.info("Convenience modules: %s.", convenience_modules) + + +def skip_public_api(app, what, name: str, obj, skip, options): + """Skip documenting members that are re-exported from the public API.""" + parts = name.split(".") + unqual = parts[-1] + if unqual in canonical_names and name != canonical_names[unqual]: + skip = True + return skip + for conv, all in convenience_modules.items(): + if unqual in all and name != f"{conv}.{unqual}": + skipper_log.warning(f"skipping {name}") + skip = True + return skip + + +def setup(sphinx): + sphinx.connect("autoapi-skip-member", skip_public_api) diff --git a/docs/source/dependencies/dependencies.rst b/docs/source/dependencies/dependencies.rst index 4b3bf1e2..88426e97 100644 --- a/docs/source/dependencies/dependencies.rst +++ b/docs/source/dependencies/dependencies.rst @@ -1,23 +1,52 @@ +.. _dependencies: Dependencies ============ -Simple actions depend only on their input parameters and the :class:`~labthings_fastapi.thing.Thing` on which they are defined. However, it's quite common to need something else, for example accessing another :class:`~labthings_fastapi.thing.Thing` instance on the same LabThings server. There are two important principles to bear in mind here: +LabThings makes use of the powerful "dependency injection" mechanism in FastAPI. You can see the `FastAPI documentation`_ for more information. In brief, FastAPI dependencies are annotated types that instruct FastAPI to supply certain function arguments automatically. This removes the need to set up resources at the start of a function, and ensures everything the function needs is declared and typed clearly. The most common use for dependencies in LabThings is where an action needs to make use of another `.Thing` on the same `.ThingServer`. -* Other :class:`~labthings_fastapi.thing.Thing` instances should be accessed using a :class:`~labthings_fastapi.client.in_server.DirectThingClient` subclass if possible. This creates a wrapper object that should work like a :class:`~labthings_fastapi.client.ThingClient`, meaning your code should work either on the server or in a client script. This makes the code much easier to debug. -* LabThings uses the FastAPI "dependency injection" mechanism, where you specify what's needed with type hints, and the argument is supplied automatically at run-time. You can see the `FastAPI documentation`_ for more information. +Inter-Thing dependencies +------------------------ + +Simple actions depend only on their input parameters and the `.Thing` on which they are defined. However, it's quite common to need something else, for example accessing another `.Thing` instance on the same LabThings server. There are two important principles to bear in mind here: -LabThings provides a shortcut to create the annotated type needed to declare a dependency on another :class:`~labthings_fastapi.thing.Thing`, with the function :func:`~labthings_fastapi.dependencies.thing.direct_thing_client_dependency`. This generates a type annotation that you can use when you define your actions, that will supply a client object when the action is called. +* Other `.Thing` instances should be accessed using a `.DirectThingClient` subclass if possible. This creates a wrapper object that should work like a `.ThingClient`, meaning your code should work either on the server or in a client script. This makes the code much easier to debug. +* LabThings uses the FastAPI "dependency injection" mechanism, where you specify what's needed with type hints, and the argument is supplied automatically at run-time. You can see the `FastAPI documentation`_ for more information. -:func:`~labthings_fastapi.dependencies.thing.direct_thing_client_dependency` takes a :func:`~labthings_fastapi.thing.Thing` class and a path as arguments: these should match the configuration of your LabThings server. Optionally, you can specify the actions that you're going to use. The default behaviour is to make all actions available, however it is more efficient to specify only the actions you will use. +In order to use on `.Thing` from another there are three steps, all shown in the example below. -Dependencies are added recursively - so if you depend on another Thing, and some of its actions have their own dependencies, those dependencies are also added to your action. Using the ``actions`` argument means you only need the dependencies of the actions you are going to use, which is more efficient. +#. Create a `.DirectThingClient` subclass for your target `.Thing`. This can be done using the `.direct_thing_client_class` function, which takes a `.Thing` subclass and a path as arguments: these should match the configuration of your LabThings server. +#. Annotate your client class with `fastapi.Depends()` to mark it as a dependency. You may assign this annotated type to a name, which is much neater when you are using it several times. +#. Use the annotated type as a type hint on one of your action's arguments. .. literalinclude:: example.py :language: python -In the example above, the :func:`increment_counter` action on :class:`TestThing` takes a :class:`MyThing` as an argument. When the action is called, the ``my_thing`` argument is supplied automatically. The argument is not the :class:`MyThing` instance, instead it is a wrapper class (a dynamically generated :class:`~labthings_fastapi.client.in_server.DirectThingClient` subclass). The wrapper should have the same signature as a :class:`~labthings_fastapi.client.ThingClient`. This means any dependencies of actions on the :class:`MyThing` are automatically supplied, so you only need to worry about the arguments that are not dependencies. The aim of this is to ensure that the code you write for your :class:`Thing` is as similar as possible to the code you'd write if you were using it through the Python client module. +In the example above, the ``increment_counter`` action on ``TestThing`` takes a ``MyThingClient`` as an argument. When the action is called, the ``my_thing`` argument is supplied automatically. The argument is not the ``MyThing`` instance, instead it is a wrapper class ``MyThingClient`` (this is a dynamically generated `.DirectThingClient` subclass). The wrapper should have the same signature as a `.ThingClient` connected to ``MyThing``. This means any dependencies of actions on the ``MyThing`` are automatically supplied, so you only need to worry about the arguments that are not dependencies. The aim of this is to ensure that the code you write for your `.Thing` is as similar as possible to the code you'd write if you were using it through the Python client module. + +.. note:: + + LabThings provides a shortcut to create the annotated type needed to declare a dependency on another `.Thing`, with the function `.direct_thing_client_dependency`. This generates a type annotation that you can use when you define your actions. + This shortcut may not work well with type checkers or linters, however, so we now recommend you declare an annotated type instead, as shown in the example. + +Dependencies are added recursively - so if you depend on another Thing, and some of its actions have their own dependencies, those dependencies are also added to your action. Using the ``actions`` argument means you only need the dependencies of the actions you are going to use, which is more efficient. If you need access to the actual Python object (e.g. you need to access methods that are not decorated as actions), you can use the :func:`~labthings_fastapi.dependencies.raw_thing.raw_thing_dependency` function instead. This will give you the actual Python object, but you will need to supply all the arguments of the actions, including dependencies, yourself. +Non-Thing dependencies +---------------------- + +LabThings provides several other dependencies, which can usually be imported directly as annotated types. For example, if your action needs to display messages as it runs, you may use an `.InvocationLogger`: + +.. code-block:: python + + import labthings_fastapi as lt + + class NoisyCounter(lt.Thing): + def count_in_logs(self, logger: lt.deps.InvocationLogger): + for i in range(10): + logger.info(f"Counter is now {i}") + +Most common dependencies can be found within `labthings_fastapi.deps`. + .. _`FastAPI documentation`: https://fastapi.tiangolo.com/tutorial/dependencies/ \ No newline at end of file diff --git a/docs/source/dependencies/example.py b/docs/source/dependencies/example.py index 89f54d7b..d170178b 100644 --- a/docs/source/dependencies/example.py +++ b/docs/source/dependencies/example.py @@ -1,15 +1,20 @@ +"""An example of how Things can use other Things via dependencies.""" + +from typing import Annotated +from fastapi import Depends import labthings_fastapi as lt from labthings_fastapi.example_things import MyThing -MyThingDep = lt.deps.direct_thing_client_dependency(MyThing, "/mything/") +MyThingClient = lt.deps.direct_thing_client_class(MyThing, "/mything/") +MyThingDep = Annotated[MyThingClient, Depends()] class TestThing(lt.Thing): - """A test thing with a counter property and a couple of actions""" + """A test thing with a counter property and a couple of actions.""" @lt.thing_action def increment_counter(self, my_thing: MyThingDep) -> None: - """Increment the counter on another thing""" + """Increment the counter on another thing.""" my_thing.increment_counter() diff --git a/docs/source/examples.rst b/docs/source/examples.rst new file mode 100644 index 00000000..77f3c611 --- /dev/null +++ b/docs/source/examples.rst @@ -0,0 +1,9 @@ +Examples +======== + +For a simple example `.Thing` and instructions on how to serve it, see the :ref:`tutorial`\ . + +For more complex examples, there is a useful collection of `.Thing` subclasses implemented as part of the `OpenFlexure Microscope`_ in the `things`_ submodule. This includes control of a camera and a translation stage, as well as various software `.Thing`\ s that integrate the two. + +.. _`OpenFlexure Microscope`: https://gitlab.com/openflexure/openflexure-microscope-server/ +.. _`things`: https://gitlab.com/openflexure/openflexure-microscope-server/-/tree/v3/src/openflexure_microscope_server/things/ \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index 65daae39..2a38b82e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -9,12 +9,14 @@ Documentation for LabThings-FastAPI wot_core_concepts.rst lt_core_concepts.rst tutorial/index.rst + examples.rst + actions.rst dependencies/dependencies.rst blobs.rst concurrency.rst using_things.rst - apidocs/index + autoapi/index `labthings-fastapi` implements a Web of Things interface for laboratory hardware using Python. This is a ground-up rewrite of python-labthings_, replacing Flask 1 and Marshmallow with FastAPI and Pydantic. It is the underlying framework for v3 of the `OpenFlexure Microscope software `_. diff --git a/docs/source/lt_core_concepts.rst b/docs/source/lt_core_concepts.rst index 7501d9b3..08ef280d 100644 --- a/docs/source/lt_core_concepts.rst +++ b/docs/source/lt_core_concepts.rst @@ -1,3 +1,5 @@ +.. _labthings_cc: + LabThings Core Concepts ======================= @@ -13,11 +15,10 @@ The server API is accessed over an HTTP requests, allowing client code (see belo Everything is a Thing --------------------- -As described in :doc:`wot_core_concepts`, a Thing represents a piece of hardware or software. LabThings-FastAPI automatically generates a `Thing Description`_ to describe each Thing. Each function offered by the Thing is either a Property or Action (LabThings-FastAPI does not yet support Events). These are termed "interaction affordances" in WoT_ terminology. +As described in :ref:`wot_cc`, a Thing represents a piece of hardware or software. LabThings-FastAPI automatically generates a :ref:`wot_td` to describe each Thing. Each function offered by the Thing is either a Property or Action (LabThings-FastAPI does not yet support Events). These are termed "interaction affordances" in WoT_ terminology. -Code on the LabThings FastAPI Server is composed of Things, however these can call generic Python functions/classes. The entire HTTP API served by the server is defined by :class:`.Thing` objects. As such the full API is composed of the actions and properties (and perhaps eventually events) defined in each Thing. +Code on the LabThings FastAPI Server is composed of Things, however these can call generic Python functions/classes. The entire HTTP API served by the server is defined by `.Thing` objects. As such the full API is composed of the actions and properties (and perhaps eventually events) defined in each Thing. -_`Thing Description`: wot_core_concepts#thing _`WoT`: wot_core_concepts Properties vs Settings @@ -28,7 +29,7 @@ A Thing in LabThings-FastAPI can have Settings as well as Properties. "Setting" Client Code ----------- -Clients or client code (Not to be confused with a :class:`.ThingClient`, see below) is the terminology used to describe any software that uses HTTP requests to access the LabThing Server. Clients can be written in any language that supports an HTTP request. However, LabThings FastAPI provides additional functionality that makes writing client code in Python easier. +Clients or client code (Not to be confused with a `.ThingClient`, see below) is the terminology used to describe any software that uses HTTP requests to access the LabThing Server. Clients can be written in any language that supports an HTTP request. However, LabThings FastAPI provides additional functionality that makes writing client code in Python easier. ThingClients ------------ diff --git a/docs/source/quickstart/quickstart.rst b/docs/source/quickstart/quickstart.rst index c91fff52..5c159af2 100644 --- a/docs/source/quickstart/quickstart.rst +++ b/docs/source/quickstart/quickstart.rst @@ -16,7 +16,7 @@ then install labthings with: :start-after: BEGIN install :end-before: END install -To define a simple example ``Thing``, paste the following into a python file, ``counter.py``: +To define a simple example ``Thing``, paste the following into a Python file, ``counter.py``: .. literalinclude:: counter.py :language: python diff --git a/docs/source/tutorial/index.rst b/docs/source/tutorial/index.rst index 3932187a..19a2f06f 100644 --- a/docs/source/tutorial/index.rst +++ b/docs/source/tutorial/index.rst @@ -1,3 +1,5 @@ +.. _tutorial: + LabThings-FastAPI tutorial ========================== diff --git a/docs/source/tutorial/running_labthings.rst b/docs/source/tutorial/running_labthings.rst index 5a42bf1a..4a1977de 100644 --- a/docs/source/tutorial/running_labthings.rst +++ b/docs/source/tutorial/running_labthings.rst @@ -1,3 +1,5 @@ +.. _tutorial_running: + Running LabThings-FastAPI ========================= @@ -15,6 +17,11 @@ Now that your server is running, you should be able to view the interactive docu Another important document is the Thing Description: this is a higher-level description of all the capabilities of each Thing in the server. For our example server, we have just one Thing, which is at ``http://127.0.0.1:5000/mything/``. This is a JSON document, but if you view it in Firefox there is a convenient tree view that makes it easier to navigate. Currently the Thing Description is not as interactive as the OpenAPI documentation, but it is rather neater as it's a higher-level description: rather than describing every possible request, it describes the capabilities of your Thing in a way that should correspond nicely to the code you might write using a Python client object, or a client in some other language. +.. _config_files: + +Configuration files +------------------- + It is worth unpicking the command you ran to start the server: it has one argument, which is a JSON string. This is fine if you are starting up a test server for one Thing, but it gets unwieldy very quickly. Most of the time, you will want to start the server with a configuration file. This is a JSON file that contains the same information as the JSON string you passed to the command above, but in a more convenient format. To do this, create a file called `example_things.json` in the same directory as your virtual environment, and put the following content in it: .. code-block:: json diff --git a/docs/source/using_things.rst b/docs/source/using_things.rst index 2b7ca2b9..ea4d24c7 100644 --- a/docs/source/using_things.rst +++ b/docs/source/using_things.rst @@ -1,9 +1,9 @@ Using Things ============ -The interface to a `Thing` is defined by its actions, properties and events [#events]_. These can all be accessed remotely via HTTP from any language, but a more convenient interface in Python is a :class:`.ThingClient` subclass. This provides a simple, pythonic interface to the :class:`.Thing`, allowing you to call actions and access properties as if they were methods and attributes of a Python object. +The interface to a `Thing` is defined by its actions, properties and events [#events]_. These can all be accessed remotely via HTTP from any language, but a more convenient interface in Python is a `.ThingClient` subclass. This provides a simple, pythonic interface to the `.Thing`, allowing you to call actions and access properties as if they were methods and attributes of a Python object. -:class:`.ThingClient` subclasses can be generated dynamically from a URL using :meth:`.ThingClient.from_url`. This creates an object with the right methods, properties and docstrings, though type hints are often missing. The client can be "introspected" to explore its methods and properties using tools that work at run-time (e.g. autocompletion in a Jupyter notebook), but "static" analysis tools will not yet work. +`.ThingClient` subclasses can be generated dynamically from a URL using :meth:`.ThingClient.from_url`. This creates an object with the right methods, properties and docstrings, though type hints are often missing. The client can be "introspected" to explore its methods and properties using tools that work at run-time (e.g. autocompletion in a Jupyter notebook), but "static" analysis tools will not yet work. .. [#events] Events are not yet implemented. @@ -17,21 +17,27 @@ _`render the interactive documentation`: https://fastapi.tiangolo.com/#interacti Dynamic class generation ------------------------- -The object returned by :meth:`.ThingClient.from_url` is an instance of a dynamically-created subclass of :class:`.ThingClient`. Dynamically creating the class is needed because we don't know what the methods and properties should be until we have downloaded the Thing Description. However, this means most code autocompletion tools, type checkers, and linters will not work well with these classes. In the future, LabThings-FastAPI will generate custom client subclasses that can be shared in client modules, which should fix these problems (see below). +The object returned by `.ThingClient.from_url` is an instance of a dynamically-created subclass of `.ThingClient`. Dynamically creating the class is needed because we don't know what the methods and properties should be until we have downloaded the Thing Description. However, this means most code autocompletion tools, type checkers, and linters will not work well with these classes. In the future, LabThings-FastAPI will generate custom client subclasses that can be shared in client modules, which should fix these problems (see below). + +.. _things_from_things: Using Things from other Things ------------------------------ -One goal of LabThings-FastAPI is to make code portable between a client (e.g. a Jupyter notebook, or a Python script on another computer) and server-side code (i.e. code inside an action of a :class:`.Thing`). This is done using a :class:`.DirectThingClient` class, which is a subclass of :class:`.ThingClient`. +One goal of LabThings-FastAPI is to make code portable between a client (e.g. a Jupyter notebook, or a Python script on another computer) and server-side code (i.e. code inside an action of a `.Thing`). This is done using a `.DirectThingClient` class, which is a subclass of `.ThingClient`. + +A `.DirectThingClient` class will call actions and properties of other `.Thing` subclasses using the same interface that would be used by a remote client, which means code for an action may be developed as an HTTP client, for example in a Jupyter notebook, and then moved to the server with minimal changes. Currently, there are a few differences in behaviour between working locally or remotely, most notably the return types (which are usually Pydantic models on the server, and currently dictionaries on the client). This should be improved in the future. + +It is also possible for a `.Thing` to access other `.Thing` instances directly. This gives access to functionality that is only available in Python, i.e. not available through a `.ThingClient` over HTTP. However, the `.Thing` must then be supplied manually with any :ref:`dependencies` required by its actions, and the public API as defined by the :ref:`wot_td` is no longer enforced. -A :class:`.DirectThingClient` class will call actions and properties of other :class:`.Thing` subclasses using the same interface that would be used by a remote client, which means code for an action may be developed as an HTTP client, for example in a Jupyter notebook, and then moved to the server with minimal changes. Currently, there are a few differences in behaviour between working locally or remotely, most notably the return types (which are usually Pydantic models on the server, and currently dictionaries on the client). This should be improved in the future. +Actions that make use of other `.Thing` objects on the same server should access them using :ref:`dependencies`. Planned future development: static code generation -------------------------------------------------- In the future, `labthings_fastapi` will generate custom client subclasses. These will have the methods and properties defined in a Python module, including type annotations. This will allow static analysis (e.g. with MyPy) and IDE autocompletion to work. Most packages that provide a `Thing` subclass will want to release a client package that is generated automatically in this way. The intention is to make it possible to add custom Python code to this client, for example to handle specialised return types more gracefully or add convenience methods. Generated client code does mean there will be more packages to install on the client in order to use a particular Thing. However, the significant benefits of having a properly defined interface should make this worthwhile. -Return types are also currently not consistent between client and server code: currently, the HTTP implementation of :class:`.ThingClient` deserialises the JSON response and returns it directly, meaning that :class:`pydantic.BaseModel` subclasses become dictionaries. This behaviour should change in the future to be consistent between client and server. Most liekly, this will mean Pydantic models are used in both cases. +Return types are also currently not consistent between client and server code: currently, the HTTP implementation of `.ThingClient` deserialises the JSON response and returns it directly, meaning that `pydantic.BaseModel` subclasses become dictionaries. This behaviour should change in the future to be consistent between client and server. Most likely, this will mean Pydantic models are used in both cases. diff --git a/docs/source/wot_core_concepts.rst b/docs/source/wot_core_concepts.rst index b7e47635..605832a1 100644 --- a/docs/source/wot_core_concepts.rst +++ b/docs/source/wot_core_concepts.rst @@ -1,27 +1,39 @@ +.. _wot_cc: + Web of Things Core Concepts =========================== LabThings is rooted in the `W3C Web of Things standards `_. Using IP networking in labs is not new, though perhaps under-used. However lack of proper standardisation has stiffled widespread adoption. LabThings, rather than try to introduce new competing standards, uses the architecture and terminology introduced by the W3C Web of Things. A full description of the core architecture can be found in the `Web of Things (WoT) Architecture `_ document. However, a brief outline of the concepts relevant to `labthings-fastapi` is given below. +.. _wot_thing: + Thing --------- -A `Thing` represents a piece of hardware or software. It could be a whole instrument (e.g. a microscope), a component within an instrument (e.g. a translation stage or camera), or a piece of software (e.g. code to tile together large area scans). `Thing`s in `labthings-fastapi` are Python classes that define Properties, Actions, and Events (see below). A Thing (sometimes called a "Web Thing") is defined by W3C as "an abstraction of a physical or a virtual entity whose metadata and interfaces are described by a WoT Thing description." +A Thing represents a piece of hardware or software. It could be a whole instrument (e.g. a microscope), a component within an instrument (e.g. a translation stage or camera), or a piece of software (e.g. code to tile together large area scans). Things in `labthings-fastapi` are Python classes that define Properties, Actions, and Events (see below). A Thing (sometimes called a "Web Thing") is defined by W3C as "an abstraction of a physical or a virtual entity whose metadata and interfaces are described by a WoT Thing description." -`labthings-fastapi` automatically generates a `Thing Description`_ to describe each `Thing`. Each function offered by the `Thing` is either a Property, Action, or Event. These are termed "interaction affordances" in WoT_ terminology. +`labthings-fastapi` automatically generates a `Thing Description`_ to describe each `.Thing`. Each function offered by the `.Thing` is either a Property, Action, or Event. These are termed "interaction affordances" in WoT_ terminology. + +.. _wot_properties: Properties ---------- As a rule of thumb, any attribute of your device that can be quickly read, or optionally written, should be a Property. For example, simple device settings, or status information (like a temperature) that takes negligible time to measure. Reading a property should never be a slow operation, as it is expected to be called frequently by clients. Properties are defined as "an Interaction Affordance that allows to read, write, or observe a state of the Thing" in the WoT_ standard. Similarly, writing to a property ought to be quick, and should not cause equipment to perform long-running operations. Properties are defined very similar to standard Python properties, using a decorator that adds them to the `Thing Description`_ and the HTTP API. +.. _wot_actions: + Actions ------- Actions generally correspond to making equipment (or software) do something. For example, starting a data acquisition, moving a stage, or changing a setting that requires a significant amount of time to complete. The key point here is that Actions are typically more complex in functionality than simply setting or getting a property. For example, they can set multiple properties simultaneously (for example, auto-exposing a camera), or they can manipulate the state of the Thing over time, for example starting a long-running data acquisition. +In WoT parlance, an Action is "invoked", and we refer to each time an Action is run as an "invocation". + `labthings-fastapi` runs actions in background threads. This allows other actions and properties to be accessed while it is running. You define actions as methods of your `Thing` class using the decorator. +.. _wot_events: + Events ------ @@ -31,5 +43,22 @@ Common examples are notifying clients when a Property is changed, or when an Act A good example of this might be having Things automatically pause data-acquisition Actions upon detection of an overheat or interlock Event from another Thing. Events are not currently implemented in `labthings-fastapi`, but are planned for future releases. +.. _wot_td: + +Thing Description +----------------- + +Each :ref:`wot_thing` is documented by a Thing Description, which is a JSON document describing all of the ways to interact with that Thing. The WoT_ standard defines the `Thing Description`_ and includes a JSON Schema against which it may be validated. + +Thing Description documents are higher-level than OpenAPI_ and focus on the capabilities of the Thing, rather than the HTTP endpoints. In general you would expect more HTTP endpoints than there are interaction affordances, so in principle client code based on a Thing Description should be more meaningful. However, OpenAPI is a much more widely adopted standard and so both forms of documentation are generated by LabThings-FastAPI. + .. _WoT: https://www.w3.org/WoT/ -.. _Thing Description: https://www.w3.org/TR/wot-thing-description/ \ No newline at end of file +.. _Thing Description: https://www.w3.org/TR/wot-thing-description/ +.. _OpenAPI: https://www.openapis.org/ + +.. _wot_affordances: + +Interaction Affordances +----------------------- + +The Web of Things standard often talks about Affordances. This is the collective term for _wot_properties, _wot_actions, and _wot_events. diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index f9db97d7..00000000 --- a/examples/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# LabThings-FastAPI Examples - -The files in this folder are example code that was used in development and may be helpful to users. It's not currently tested, so there are no guarantees as to how current each example is. Some of them have been moved into `/tests/` and those ones do get checked: at some point in the future a combined documentation/testing system might usefully deduplicate this. - -Two camera-related examples have been removed, there are better `Thing`s already written for handling cameras as part of the [OpenFlexure Microscope] and you can find the relevant [camera Thing code] there. - -To run these examples, it's best to look at the tutorial or quickstart guides on our [readthedocs] site. - -[readthedocs]: https://labthings-fastapi.readthedocs.io/ -[OpenFlexure Microscope]: https://openflexure.org/ -[camera Thing code]: https://gitlab.com/openflexure/openflexure-microscope-server/-/tree/v3/src/openflexure_microscope_server/things/camera?ref_type=heads diff --git a/examples/counter.py b/examples/counter.py deleted file mode 100644 index 95438422..00000000 --- a/examples/counter.py +++ /dev/null @@ -1,32 +0,0 @@ -import time - -import labthings_fastapi as lt - - -class TestThing(lt.Thing): - """A test thing with a counter property and a couple of actions""" - - @lt.thing_action - def increment_counter(self) -> None: - """Increment the counter property - - This action doesn't do very much - all it does, in fact, - is increment the counter (which may be read using the - `counter` property). - """ - self.counter += 1 - - @lt.thing_action - def slowly_increase_counter(self) -> None: - """Increment the counter slowly over a minute""" - for i in range(60): - time.sleep(1) - self.increment_counter() - - counter = lt.ThingProperty( - model=int, initial_value=0, readonly=True, description="A pointless counter" - ) - - -server = lt.ThingServer() -server.add_thing(TestThing(), "/test") diff --git a/examples/demo_thing_server.py b/examples/demo_thing_server.py deleted file mode 100644 index 17166048..00000000 --- a/examples/demo_thing_server.py +++ /dev/null @@ -1,129 +0,0 @@ -import logging -import time -from typing import Optional, Annotated -from pydantic import Field -from fastapi.responses import HTMLResponse - -import labthings_fastapi as lt - -logging.basicConfig(level=logging.INFO) - - -class MyThing(lt.Thing): - @lt.thing_action - def anaction( - self, - repeats: Annotated[ - int, Field(description="The number of times to try the action") - ], - undocumented: int, - title: Annotated[ - str, Field(description="the title of the invocation") - ] = "Untitled", - attempts: Annotated[ - Optional[list[str]], - Field( - description="Names for each attempt - I suggest final, Final, FINAL." - ), - ] = None, - ) -> dict[str, str]: - """Quite a complicated action - - This action has lots of parameters and is designed to confuse my schema - generator. I hope it doesn't! - - I might even use some Markdown here: - - * If this renders, it supports lists - * With at east two items. - """ - # We should be able to call actions as normal Python functions - self.increment_counter() - return "finished!!" - - @lt.thing_action - def increment_counter(self): - """Increment the counter property - - This action doesn't do very much - all it does, in fact, - is increment the counter (which may be read using the - `counter` property). - """ - self.counter += 1 - - @lt.thing_action - def slowly_increase_counter(self): - """Increment the counter slowly over a minute""" - for i in range(60): - time.sleep(1) - self.increment_counter() - - counter = lt.ThingProperty( - model=int, initial_value=0, readonly=True, description="A pointless counter" - ) - - foo = lt.ThingProperty( - model=str, - initial_value="Example", - description="A pointless string for demo purposes.", - ) - - -thing_server = lt.ThingServer() -my_thing = MyThing() -td = my_thing.thing_description() -my_thing.validate_thing_description() -thing_server.add_thing(my_thing, "/my_thing") - -app = thing_server.app - - -html = """ - - - - Chat - - -

WebSocket Chat

-
- - -
-
    -
- - - -""" - -""" -{"messageType":"addPropertyObservation","data":{"foo":true, "counter":true}} -{"messageType":"addActionObservation","data":{"increment_counter":true}} -""" - - -@app.get("/wsclient", tags=["websockets"]) -async def get(): - return HTMLResponse(html) - - -if __name__ == "__main__": - import uvicorn - - uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/pyproject.toml b/pyproject.toml index 42217188..58e01fea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,15 @@ dev = [ "ruff>=0.1.3", "types-jsonschema", "Pillow", + "flake8", + "flake8-pyproject", + "flake8-docstrings", + "flake8-rst", + "flake8-rst-docstrings", + "pydoclint[flake8]", + "sphinx-rtd-theme", + "sphinx>=7.2", + "sphinx-autoapi", ] [project.urls] @@ -62,8 +71,54 @@ addopts = [ [tool.ruff] target-version = "py310" +[tool.ruff.format] +docstring-code-format = true + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "D", "DOC"] +ignore = [ + "D203", # incompatible with D204 + "D213", # incompatible with D212 + "DOC402", # doesn't work with sphinx-style docstrings, use pydoclint + "DOC201", # doesn't work with sphinx-style docstrings, use pydoclint + "DOC501", # doesn't work with sphinx-style docstrings, use pydoclint + "DOC502", # doesn't work with sphinx-style docstrings, use pydoclint +] +preview = true + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["D", "DOC"] +"docs/*" = ["D", "DOC"] + [tool.mypy] plugins = ["pydantic.mypy", "numpy.typing.mypy_plugin"] +[tool.flake8] +extend-ignore = [ + "DOC301", # allow class + __init__ docstrings + "D202", # conflicts with ruff format +] +max-line-length = 88 +rst-roles = [ + "class", + "func", + "mod", + "ref", + "deco", + "doc", +] +rst-directives = [ + "todo", +] +style = "sphinx" +arg-type-hints-in-docstring = false +skip-checking-short-docstrings = false +allow-init-docstring = true +check-style-mismatch = true +show-filenames-in-every-violation-message = true +check-return-types = false +check-class-attributes = false # prefer docstrings on the attributes +check-yield-types = false # use type annotations instead + [project.scripts] -labthings-server = "labthings_fastapi.server.cli:serve_from_cli" \ No newline at end of file +labthings-server = "labthings_fastapi.server.cli:serve_from_cli" diff --git a/src/labthings_fastapi/__init__.py b/src/labthings_fastapi/__init__.py index d15b6df4..ec969f65 100644 --- a/src/labthings_fastapi/__init__.py +++ b/src/labthings_fastapi/__init__.py @@ -1,3 +1,24 @@ +r"""LabThings-FastAPI. + +This is the top level module for LabThings-FastAPI, a library for building +:ref:`wot_cc` devices using Python. There is documentation on readthedocs_, +and the recommended place to start is :doc:`index`\ . + +.. _readthedocs: https://labthings-fastapi.readthedocs.io/ + +This module contains a number of convenience +imports and is intended to be imported using: + +.. code-block:: python + + import labthings_fastapi as lt + +The example code elsewhere in the documentation generally follows this +convention. Symbols in the top-level module mostly exist elsewhere in +the package, but should be imported from here as a preference, to ensure +code does not break if modules are rearranged. +""" + from .thing import Thing from .descriptors import ThingProperty, ThingSetting from .decorators import ( diff --git a/src/labthings_fastapi/actions/__init__.py b/src/labthings_fastapi/actions/__init__.py index 7532a87e..3cd6324e 100644 --- a/src/labthings_fastapi/actions/__init__.py +++ b/src/labthings_fastapi/actions/__init__.py @@ -1,8 +1,24 @@ +"""Actions module. + +:ref:`wot_actions` are represented by methods, decorated with the `.thing_action` +decorator. + +See the :ref:`actions` documentation for a top-level overview of actions in +LabThings-FastAPI. + +Developer notes +--------------- + +Currently much of the code related to Actions is in `.thing_action` and the +underlying `.ActionDescriptor`. This is likely to be refactored in the near +future. +""" + from __future__ import annotations import datetime import logging from collections import deque -from threading import Event, Thread, Lock +from threading import Thread, Lock from typing import MutableSequence, Optional, Any import uuid from typing import TYPE_CHECKING @@ -12,8 +28,8 @@ from ..utilities import model_to_dict from ..utilities.introspection import EmptyInput -from ..thing_description.model import LinkElement -from .invocation_model import InvocationModel, InvocationStatus +from ..thing_description._model import LinkElement +from .invocation_model import InvocationModel, InvocationStatus, LogRecordModel from ..dependencies.invocation import ( CancelHook, InvocationCancelledError, @@ -27,26 +43,53 @@ from ..thing import Thing ACTION_INVOCATIONS_PATH = "/action_invocations" +"""The API route used to list `.Invocation` objects.""" class Invocation(Thread): - """A Thread subclass that retains output values and tracks progress + """A Thread subclass that retains output values and tracks progress. + + `.Invocation` threads add several bits of functionality compared to the base + `threading.Thread`. - TODO: In the future this should probably not be a Thread subclass, but might run in - a thread anyway. + * They are instantiated with an `.ActionDescriptor` and a `.Thing` + rather than a target function (see ``__init__``). + * Each invocation is assigned a unique ``ID`` to allow it to be polled + over HTTP. + * A `.CancelHook` is provided to allow the invocation to stop gracefully + if it is cancelled by the user. """ def __init__( self, action: ActionDescriptor, thing: Thing, + id: uuid.UUID, input: Optional[BaseModel] = None, dependencies: Optional[dict[str, Any]] = None, - default_stop_timeout: float = 5, log_len: int = 1000, - id: Optional[uuid.UUID] = None, cancel_hook: Optional[CancelHook] = None, ): + """Create a thread to run an action and track its outputs. + + :param action: provides the function that we run, as well as metadata + and type information. The descriptor is not bound to an object, so we + supply the `.Thing` it's bound to when the function is run. + :param thing: is the object on which we are running the ``action``, i.e. + it is supplied to the function wrapped by ``action`` as the ``self`` + argument. + :param id: is a `uuid.UUID` used to identify the invocation, for example + when polling its status via HTTP. + :param input: is a `pydantic.BaseModel` representing the body of the HTTP + request that invoked the action. It is supplied to the function as + keyword arguments. + :param dependencies: is a dictionary of keyword arguments, supplied by + FastAPI by its dependency injection mechanism. + :param log_len: sets the number of log entries that will be held in + memory by the invocation's logger. + :param cancel_hook: is a `threading.Event` subclass that tells the + invocation it's time to stop. See `.CancelHook`. + """ Thread.__init__(self, daemon=True) # keep track of the corresponding ActionDescriptor @@ -57,11 +100,7 @@ def __init__( self.cancel_hook = cancel_hook # A UUID for the Invocation (not the same as the threading.Thread ident) - self._ID = id if id is not None else uuid.uuid4() # Task ID - - # Event to track if the user has requested stop - self.stopping: Event = Event() - self.default_stop_timeout: float = default_stop_timeout + self._ID = id # Task ID # How long to keep the invocation after it finishes self.retention_time = action.retention_time @@ -79,61 +118,65 @@ def __init__( @property def id(self) -> uuid.UUID: - """ - UUID for the thread. Note this not the same as the native thread ident. - """ + """UUID for the thread. Note this not the same as the native thread ident.""" return self._ID @property def output(self) -> Any: - """ - Return value of the Action. If the Action is still running, returns None. - """ + """Return value of the Action. If the Action is still running, returns None.""" with self._status_lock: return self._return_value @property - def log(self): + def log(self) -> list[LogRecordModel]: """A list of log items generated by the Action.""" with self._status_lock: return list(self._log) @property def status(self) -> InvocationStatus: - """ - Current running status of the thread. - - ============== ============================================= - Status Meaning - ============== ============================================= - ``pending`` Not yet started - ``running`` Currently in-progress - ``completed`` Finished without error - ``cancelled`` Thread stopped after a cancel request - ``error`` Exception occurred in thread - ============== ============================================= + """Current running status of the thread. + + See `.InvocationStatus` for the values and their meanings. """ with self._status_lock: return self._status @property def action(self): - return self.action_ref() + """The `.ActionDescriptor` object running in this thread.""" + action = self.action_ref() + assert action is not None, "The action for an `Invocation` has been deleted!" + return action @property - def thing(self): - return self.thing_ref() + def thing(self) -> Thing: + """The `.Thing` to which the action is bound, i.e. this is ``self``.""" + thing = self.thing_ref() + assert thing is not None, "The `Thing` on which an action was run is missing!" + return thing - def cancel(self): - """Cancel the task by requesting the code to stop + def cancel(self) -> None: + """Cancel the task by requesting the code to stop. - This is very much not guaranteed to work: the action must use - a CancelHook dependency and periodically check it. + This is an opt-in feature: the action must use + a `.CancelHook` dependency and periodically check it. """ if self.cancel_hook is not None: self.cancel_hook.set() - def response(self, request: Optional[Request] = None): + def response(self, request: Optional[Request] = None) -> InvocationModel: + """Generate a representation of the invocation suitable for HTTP. + + When an invocation is polled, we return a JSON object that includes + its status, any log entries, a return value (if completed), and a link + to poll for updates. + + :param request: is used to generate the ``href`` in the response, which + should retrieve an updated version of this response. + + :return: an `.InvocationModel` representing this `.Invocation`. + """ if request: href = str(request.url_for("action_invocation", id=self.id)) else: @@ -156,8 +199,36 @@ def response(self, request: Optional[Request] = None): log=self.log, ) - def run(self): - """Overrides default threading.Thread run() method""" + def run(self) -> None: + """Run the action and track progress. + + `.Invocation` overrides the default `threading.Thread.run` method to + add ways to track its progress and capture the return value. + + The code to be run is the function wrapped in the `.ActionDescriptor` + that is passed in as ``action``. Its arguments are the associated + `.Thing` (the first argument, i.e. ``self``), the ``input`` model + (split into keyword arguments for each field), and any ``dependencies`` + (also as keyword arguments). + + We update the status of the action by setting ``self._status`` and + emitting a changed event. This runs async code in the event loop that + informs any clients listening over websockets that the event's status + has changed. + + Logs are retained by a custom log handler, and are included when the + `.Invocation` is serialised over HTTP. + + If exceptions are raised by the action code, these are caught and + stored. The status is then set to ERROR and the thread terminates. + + See `.Invocation.status` for status values. + + :raise Exception: any exception raised in the action function will + propagate through this method. Usually, this will just cause the + thread to terminate after setting ``status`` to ``ERROR`` and + saving the exception to ``self._exception``. + """ try: self.action.emit_changed_event(self.thing, self._status) @@ -207,50 +278,60 @@ def run(self): class DequeLogHandler(logging.Handler): + """A log handler that stores entries in memory.""" + def __init__( self, dest: MutableSequence, - level=logging.INFO, + level: int = logging.INFO, ): - """Set up a log handler that appends messages to a list. - - This log handler will first filter by ``thread``, if one is - supplied. This should be a ``threading.Thread`` object. - Only log entries from the specified thread will be - saved. + """Set up a log handler that appends messages to a deque. - ``dest`` should specify a deque, to which we will append - each log entry as it comes in. This is assumed to be thread - safe. - - NB this log handler does not currently rotate or truncate - the list - so if you use it on a thread that produces a - lot of log messages, you may run into memory problems. + .. warning:: + This log handler does not currently rotate or truncate + the list - so if you use it on a thread that produces a + lot of log messages, you may run into memory problems. + Using a `.deque` with a finite capacity helps to mitigate + this. + :param dest: should specify a deque, to which we will append + each log entry as it comes in. This is assumed to be thread + safe. + :param level: sets the level of the logger. For most invocations, + a log level of `logging.INFO` is appropriate. """ logging.Handler.__init__(self) self.setLevel(level) self.dest = dest - def emit(self, record): - """Save a log record to the destination deque""" + def emit(self, record: logging.LogRecord) -> None: + """Save a log record to the destination deque. + + :param record: the `logging.LogRecord` object to add. + """ self.dest.append(record) class ActionManager: - """A class to manage a collection of actions""" + """A class to manage a collection of actions.""" def __init__(self): + """Set up an `.ActionManager`.""" self._invocations = {} self._invocations_lock = Lock() @property - def invocations(self): + def invocations(self) -> list[Invocation]: + """A list of all the `.Invocation` objects running or recently completed.""" with self._invocations_lock: return list(self._invocations.values()) - def append_invocation(self, invocation: Invocation): + def append_invocation(self, invocation: Invocation) -> None: + """Add an `.Invocation` to the `.ActionManager`. + + :param invocation: The `.Invocation` to add. + """ with self._invocations_lock: self._invocations[invocation.id] = invocation @@ -263,7 +344,28 @@ def invoke_action( dependencies: dict[str, Any], cancel_hook: CancelHook, ) -> Invocation: - """Invoke an action, returning the thread where it's running""" + """Invoke an action, returning the thread where it's running. + + See `.Invocation` for more details. + + :param action: provides the function that we run, as well as metadata + and type information. The descriptor is not bound to an object, so we + supply the `.Thing` it's bound to when the function is run. + :param thing: is the object on which we are running the ``action``, i.e. + it is supplied to the function wrapped by ``action`` as the ``self`` + argument. + :param id: is a `uuid.UUID` used to identify the invocation, for example + when polling its status via HTTP. + :param input: is a `pydantic.BaseModel` representing the body of the HTTP + request that invoked the action. It is supplied to the function as + keyword arguments. + :param dependencies: is a dictionary of keyword arguments, supplied by + FastAPI by its dependency injection mechanism. + :param cancel_hook: is a `threading.Event` subclass that tells the + invocation it's time to stop. See `.CancelHook`. + + :return: an `.Invocation` object that has been started. + """ thread = Invocation( action=action, thing=thing, @@ -277,7 +379,11 @@ def invoke_action( return thread def get_invocation(self, id: uuid.UUID) -> Invocation: - """Retrieve an invocation by ID""" + """Retrieve an invocation by ID. + + :param id: the unique ID of the action to retrieve. + :return: the `.Invocation` object. + """ with self._invocations_lock: return self._invocations[id] @@ -285,19 +391,38 @@ def list_invocations( self, action: Optional[ActionDescriptor] = None, thing: Optional[Thing] = None, - as_responses: bool = False, request: Optional[Request] = None, ) -> list[InvocationModel]: - """All of the invocations currently managed""" + """All of the invocations currently managed. + + Returns a list of `.InvocationModel` instances representing all the + invocations that are currently running, or have recently completed and + not yet expired. + + :param action: filters out only the invocations of a particular + `.ActionDescriptor`. Note that if there are two Things + of the same subclass, filtering by action will return invocations + on either `.Thing`. + :param thing: returns only invocations of actions on a particular `.Thing`. + This will often be combined with filtering by ``action`` to give the + list of invocations returned by a GET request on an action endpoint. + :param request: is used to pass a `fastapi.Request` object to the + `.Invocation.response` method. Doing so ensures the URL returned as + ``href`` in the response matches the address used to communicate with + the server (i.e. it uses `fastapi.Request.url_for` instead of a path + generated from a string). + + :return: A list of invocations, optionally filtered by Thing and/or Action. + """ return [ - i.response(request=request) if as_responses else i + i.response(request=request) for i in self.invocations if thing is None or i.thing == thing if action is None or i.action == action ] def expire_invocations(self): - """Delete invocations that have passed their expiry time""" + """Delete invocations that have passed their expiry time.""" to_delete = [] with self._invocations_lock: for k, v in self._invocations.items(): @@ -308,12 +433,15 @@ def expire_invocations(self): for k in to_delete: del self._invocations[k] - def attach_to_app(self, app: FastAPI): - """Add /action_invocations and /action_invocation/{id} endpoints to FastAPI""" + def attach_to_app(self, app: FastAPI) -> None: + """Add /action_invocations and /action_invocation/{id} endpoints to FastAPI. + + :param app: The `fastapi.FastAPI` application to which we add the endpoints. + """ @app.get(ACTION_INVOCATIONS_PATH, response_model=list[InvocationModel]) def list_all_invocations(request: Request, _blob_manager: BlobIOContextDep): - return self.list_invocations(as_responses=True, request=request) + return self.list_invocations(request=request) @app.get( ACTION_INVOCATIONS_PATH + "/{id}", @@ -322,7 +450,20 @@ def list_all_invocations(request: Request, _blob_manager: BlobIOContextDep): ) def action_invocation( id: uuid.UUID, request: Request, _blob_manager: BlobIOContextDep - ): + ) -> InvocationModel: + """Return a description of a specific action. + + :param id: The action's ID (from the path). + :param request: FastAPI dependency for the request object, used to + find URLs via ``url_for``. + :param _blob_manager: FastAPI dependency that enables `.Blob` objects + to be serialised. + + :return: Details of the invocation. + + :raise HTTPException: with code ``404`` if the invocation is not + found. + """ try: with self._invocations_lock: return self._invocations[id].response(request=request) @@ -347,10 +488,20 @@ def action_invocation( }, ) def action_invocation_output(id: uuid.UUID, _blob_manager: BlobIOContextDep): - """Get the output of an action invocation + """Get the output of an action invocation. This returns just the "output" component of the action invocation. If the output is a file, it will return the file. + + :param id: The action's ID (from the path). + :param _blob_manager: FastAPI dependency that enables `.Blob` objects + to be serialised. + + :return: The output of the invocation, as a `pydantic.BaseModel` + instance. If this is a `.Blob`, it may be returned directly. + + :raise HTTPException: with code ``404`` if the invocation is not + found. """ with self._invocations_lock: try: @@ -384,7 +535,13 @@ def action_invocation_output(id: uuid.UUID, _blob_manager: BlobIOContextDep): }, ) def delete_invocation(id: uuid.UUID) -> None: - """Cancel an action invocation""" + """Cancel an action invocation. + + :param id: The unique ID of the invocation to cancel (from the URL). + + :raise HTTPException: with code ``404`` if the invocation is not + found, or ``503`` if the invocation is not currently running. + """ with self._invocations_lock: try: invocation: Any = self._invocations[id] diff --git a/src/labthings_fastapi/actions/invocation_model.py b/src/labthings_fastapi/actions/invocation_model.py index 6c0c357e..f6ad6a50 100644 --- a/src/labthings_fastapi/actions/invocation_model.py +++ b/src/labthings_fastapi/actions/invocation_model.py @@ -1,3 +1,8 @@ +"""Invocation Model. + +This module contains types used to describe an `.Invocation`. +""" + from __future__ import annotations from datetime import datetime from enum import Enum @@ -7,19 +12,26 @@ from pydantic import BaseModel, ConfigDict, model_validator -from ..thing_description.model import Links +from ..thing_description._model import Links class InvocationStatus(Enum): + """The current status of an `.Invocation`.""" + PENDING = "pending" + """The `.Invocation` has not yet been started.""" RUNNING = "running" + """The `.Invocation` is running in its thread.""" COMPLETED = "completed" + """The `.Invocation` finished successfully. A return value may be available.""" CANCELLED = "cancelled" + """The `.Invocation` was cancelled and has finished.""" ERROR = "error" + """The `.Invocation` terminated unexpectedly due to an error.""" class LogRecordModel(BaseModel): - """A model to serialise logging.LogRecord objects""" + """A model to serialise `logging.LogRecord` objects.""" model_config = ConfigDict(from_attributes=True, extra="ignore") @@ -32,7 +44,13 @@ class LogRecordModel(BaseModel): @model_validator(mode="before") @classmethod - def generate_message(cls, data: Any): + def generate_message(cls, data: Any) -> Any: + """Ensure LogRecord objects have constructed their message. + + :param data: The LogRecord to process. + + :return: The LogRecord, with a message constructed. + """ if not hasattr(data, "message"): if isinstance(data, logging.LogRecord): try: @@ -52,6 +70,14 @@ def generate_message(cls, data: Any): class GenericInvocationModel(BaseModel, Generic[InputT, OutputT]): + """A model to serialise `.Invocation` objects when they are polled over HTTP. + + The input and output models are generic parameters, to allow this model to + be used for specific Actions. These are usually set to `Any` because the + invocation endpoint is not specific to any one Action, and thus the types + are not known in advance. + """ + status: InvocationStatus id: uuid.UUID action: str diff --git a/src/labthings_fastapi/client/__init__.py b/src/labthings_fastapi/client/__init__.py index 9bd8ed25..c7345869 100644 --- a/src/labthings_fastapi/client/__init__.py +++ b/src/labthings_fastapi/client/__init__.py @@ -1,8 +1,8 @@ -"""A first pass at a client library for LabThings-FastAPI +"""Code to access `.Thing` features over HTTP. -This will become its own package if it's any good. The goal is to see if we can -make a client library that produces introspectable Python objects from a Thing -Description. +This module defines a base class for controlling LabThings-FastAPI over HTTP. +It is based on `httpx`, and attempts to create a simple wrapper such that +each Action becomes a method and each Property becomes an attribute. """ from __future__ import annotations @@ -17,45 +17,119 @@ from .outputs import ClientBlobOutput - +__all__ = ["ThingClient", "poll_invocation"] ACTION_RUNNING_KEYWORDS = ["idle", "pending", "running"] -def get_link(obj: dict, rel: str) -> Mapping: - """Retrieve a link from an object's `links` list, by its `rel` attribute""" - return next(link for link in obj["links"] if link["rel"] == rel) +class ObjectHasNoLinksError(KeyError): + """We attempted to use the `links` key but it was not there. + + `links` is used in several places, including in the representation of + `.Invocation` objects. It should be a list of dictionaries, each of + which represents a link, with `href` and `rel` keys. + """ + + +def _get_link(obj: dict, rel: str) -> Mapping: + """Retrieve a link from an object's ``links`` list, by its ``rel`` attribute. + + Various places in the :ref:`wot_td` feature a list of links. This is represented + in JSON as a property called ``links`` which is a list of objects that have + ``href`` and ``rel`` properties. + + This function takes an object (which deserialises to a ``dict`` in Python) + and looks for its ``links`` item, then iterates through the objects there + to find the first one with a particular ``rel`` value. For example, we + use this to find the ``self`` link on an invocation. + + :param obj: the deserialised JSON response from querying an invocation. + this should be a dictionary containing at least a ``links`` key, which + is a list of dictionaries, each with ``href`` and ``rel`` defined. + :param rel: the value of the ``rel`` key in the link we are looking for. + + :return: a dictionary representing the link. It should contain at least + ``href`` and ``rel`` keys. + + :raise ObjectHasNoLinksError: if there is no ``links`` item. + :raise KeyError: if there is no link with the specified ``rel`` value. + """ + if "links" not in obj: + raise ObjectHasNoLinksError(f"Can't find any links on {obj}.") + try: + return next(link for link in obj["links"] if link["rel"] == rel) + except StopIteration: + raise KeyError(f"No link was found with rel='{rel}' on {obj}.") -def get_link_href(obj: dict, rel: str) -> str: - """Retrieve the `href` from an object's `links` list, by its `rel` attribute""" - return get_link(obj, rel)["href"] +def invocation_href(invocation: dict) -> str: + """Extract the endpoint address from an invocation dictionary. + :param invocation: The invocation's dictionary representation, i.e. the + deserialised JSON response from starting or polling an action. -def task_href(t): - """Extract the endpoint address from a task dictionary""" - return get_link(t, "self")["href"] + :return: The `href` value to poll the invocation. + .. note:: -def poll_task(client, task, interval=0.5, first_interval=0.05): - """Poll a task until it finishes, and return the return value""" + Exceptions may propagate from `._get_link`. + """ + return _get_link(invocation, "self")["href"] + + +def poll_invocation( + client: httpx.Client, + invocation: dict, + interval: float = 0.5, + first_interval: float = 0.05, +) -> dict: + """Poll a invocation until it finishes, and return the output. + + When actions are invoked in a LabThings-FastAPI server, the + initial POST request returns immediately. The returned invocation + includes a link that may be polled to find out when the action + has completed, whether it was successful, and retrieve its + output. + + :param client: the `httpx.Client` to use for HTTP requests. + :param invocation: the dictionary returned from the initial POST request. + :param interval: sets how frequently we poll, in seconds. + :param first_interval: sets how long we wait before the first + polling request. Often, it makes sense for this to be a short + interval, in case the action fails (or returns) immediately. + + :return: the completed invocation as a dictionary. + """ first_time = True - while task["status"] in ACTION_RUNNING_KEYWORDS: + while invocation["status"] in ACTION_RUNNING_KEYWORDS: time.sleep(first_interval if first_time else interval) - r = client.get(task_href(task)) + r = client.get(invocation_href(invocation)) r.raise_for_status() - task = r.json() + invocation = r.json() first_time = False - return task + return invocation class ThingClient: - """A client for a LabThings-FastAPI Thing + """A client for a LabThings-FastAPI Thing. + + .. note:: + ThingClient must be subclassed to add actions/properties, + so this class will be minimally useful on its own. - NB ThingClient must be subclassed to add actions/properties, - so this class will be minimally useful on its own. + The best way to get a client for a particular Thing is + currently `.ThingClient.from_url`, which dynamically + creates a subclass with the right attributes. """ def __init__(self, base_url: str, client: Optional[httpx.Client] = None): + """Create a ThingClient connected to a remote Thing. + + :param base_url: the base URL of the Thing. This should be the URL + of the Thing Description document. + :param client: an optional `httpx.Client` object to use for all + HTTP requests. This may be a `fastapi.TestClient` object for + testing purposes. + """ parsed = urlparse(base_url) server = f"{parsed.scheme}://{parsed.netloc}" self.server = server @@ -63,65 +137,119 @@ def __init__(self, base_url: str, client: Optional[httpx.Client] = None): self.client = client or httpx.Client(base_url=server) def get_property(self, path: str) -> Any: + """Make a GET request to retrieve the value of a property. + + :param path: the URI of the ``getproperty`` endpoint, relative + to the ``base_url``. + + :return: the property's value, as deserialised from JSON. + """ r = self.client.get(urljoin(self.path, path)) r.raise_for_status() return r.json() def set_property(self, path: str, value: Any): + """Make a PUT request to set the value of a property. + + :param path: the URI of the ``getproperty`` endpoint, relative + to the ``base_url``. + :param value: the property's value. Currently this must be + serialisable to JSON. + """ r = self.client.put(urljoin(self.path, path), json=value) r.raise_for_status() def invoke_action(self, path: str, **kwargs): - "Invoke an action on the Thing" + r"""Invoke an action on the Thing. + + This method will make the initial POST request to invoke an action, + then poll the resulting invocation until it completes. If successful, + the action's output will be returned directly. + + While the action is running, log messages will be re-logged locally. + If you have enabled logging to the console, these should be visible. + + :param path: the URI of the ``invokeaction`` endpoint, relative to the + ``base_url`` + :param \**kwargs: Additional arguments will be combined into the JSON + body of the ``POST`` request and sent as input to the action. + These will be validated on the server. + + :return: the output value of the action. + + :raise RuntimeError: is raised if the action does not complete successfully. + """ for k in kwargs.keys(): if isinstance(kwargs[k], ClientBlobOutput): kwargs[k] = {"href": kwargs[k].href, "media_type": kwargs[k].media_type} r = self.client.post(urljoin(self.path, path), json=kwargs) r.raise_for_status() - task = poll_task(self.client, r.json()) - if task["status"] == "completed": + invocation = poll_invocation(self.client, r.json()) + if invocation["status"] == "completed": if ( - isinstance(task["output"], Mapping) - and "href" in task["output"] - and "media_type" in task["output"] + isinstance(invocation["output"], Mapping) + and "href" in invocation["output"] + and "media_type" in invocation["output"] ): return ClientBlobOutput( - media_type=task["output"]["media_type"], - href=task["output"]["href"], + media_type=invocation["output"]["media_type"], + href=invocation["output"]["href"], client=self.client, ) - return task["output"] + return invocation["output"] else: - raise RuntimeError(f"Action did not complete successfully: {task}") + raise RuntimeError(f"Action did not complete successfully: {invocation}") def follow_link(self, response: dict, rel: str) -> httpx.Response: - """Follow a link in a response object, by its `rel` attribute""" - href = get_link_href(response, rel) + """Follow a link in a response object, by its `rel` attribute. + + :param response: is the dictionary returned by e.g. `.poll_invocation`. + :param rel: picks the link to follow by matching its ``rel`` + item. + + :return: the response to making a ``GET`` request to the link. + """ + href = _get_link(response, rel)["href"] r = self.client.get(href) r.raise_for_status() return r @classmethod - def from_url( - cls, thing_url: str, client: Optional[httpx.Client] = None, **kwargs - ) -> Self: - """Create a ThingClient from a URL + def from_url(cls, thing_url: str, client: Optional[httpx.Client] = None) -> Self: + """Create a ThingClient from a URL. This will dynamically create a subclass with properties and actions, and return an instance of that subclass pointing at the Thing URL. - Additional `kwargs` will be passed to the subclass constructor, in - particular you may pass a `client` object (useful for testing). + :param thing_url: The base URL of the Thing, which should also be the + URL of its Thing Description. + :param client: is an optional `httpx.Client` object. If not present, + one will be created. This is particularly useful if you need to + set HTTP options, or if you want to work with a local server + object for testing purposes (see `fastapi.TestClient`). + + :return: a `.ThingClient` subclass with properties and methods that + match the retrieved Thing Description (see :ref:`wot_thing`). """ td_client = client or httpx r = td_client.get(thing_url) r.raise_for_status() subclass = cls.subclass_from_td(r.json()) - return subclass(thing_url, client=client, **kwargs) + return subclass(thing_url, client=client) @classmethod def subclass_from_td(cls, thing_description: dict) -> type[Self]: - """Create a ThingClient subclass from a Thing Description""" + """Create a ThingClient subclass from a Thing Description. + + Dynamically subclass `.ThingClient` to add properties and + methods for each property and action in the Thing Description. + + :param thing_description: A :ref:`wot_td` as a dictionary, which will + be used to construct the class. + + :return: a `.ThingClient` subclass with the right properties and + methods. + """ my_thing_description = thing_description class Client(cls): # type: ignore[valid-type, misc] @@ -140,6 +268,8 @@ class Client(cls): # type: ignore[valid-type, misc] class PropertyClientDescriptor: + """A base class for properties on `.ThingClient` objects.""" + pass @@ -151,7 +281,28 @@ def property_descriptor( writeable: bool = True, property_path: Optional[str] = None, ) -> PropertyClientDescriptor: - """Create a correctly-typed descriptor that gets and/or sets a property""" + """Create a correctly-typed descriptor that gets and/or sets a property. + + The returned `.PropertyClientDescriptor` will have ``__get__`` and + (optionally) ``__set__`` methods that are typed according to the + supplied ``model``. The descriptor should be added to a `.ThingClient` + subclass and used to access the relevant property via + `.ThingClient.get_property` and `.ThingClient.set_property`. + + :param property_name: should be the name of the property (i.e. the + name it takes in the thing description, and also the name it is + assigned to in the class). + :param model: the Python ``type`` or a ``pydantic.BaseModel`` that + represents the datatype of the property. + :param description: text to use for a docstring. + :param readable: whether the property may be read (i.e. has ``__get__``). + :param writeable: whether the property may be written to. + :param property_path: the URL of the ``getproperty`` and ``setproperty`` + HTTP endpoints. Currently these must both be the same. These are + relative to the ``base_url``, i.e. the URL of the Thing Description. + + :return: a descriptor allowing access to the specified property. + """ class P(PropertyClientDescriptor): name = property_name @@ -183,8 +334,20 @@ def __set__(self, obj: ThingClient, value: Any): return P() -def add_action(cls: type[ThingClient], action_name: str, action: dict): - """Add an action to a ThingClient subclass""" +def add_action(cls: type[ThingClient], action_name: str, action: dict) -> None: + """Add an action to a ThingClient subclass. + + A method will be added to the class that calls the provided action. + Currently, this will have a return type hint but no argument names + or type hints. + + :param cls: the `.ThingClient` subclass to which we are adding the + action. + :param action_name: is both the name we assign the method to, and + the name of the action in the Thing Description. + :param action: a dictionary representing the action, in :ref:`wot_td` + format. + """ def action_method(self, **kwargs): return self.invoke_action(action_name, **kwargs) @@ -197,7 +360,20 @@ def action_method(self, **kwargs): def add_property(cls: type[ThingClient], property_name: str, property: dict): - """Add a property to a ThingClient subclass""" + """Add a property to a ThingClient subclass. + + A descriptor will be added to the provided class that makes the + attribute ``property_name`` get and/or set the property described + by the ``property`` dictionary. + + + :param cls: the `.ThingClient` subclass to which we are adding the + property. + :param property_name: is both the name we assign the descriptor to, and + the name of the property in the Thing Description. + :param property: a dictionary representing the property, in :ref:`wot_td` + format. + """ setattr( cls, property_name, diff --git a/src/labthings_fastapi/client/in_server.py b/src/labthings_fastapi/client/in_server.py index 018e9d5c..b5435bff 100644 --- a/src/labthings_fastapi/client/in_server.py +++ b/src/labthings_fastapi/client/in_server.py @@ -1,10 +1,15 @@ """A mock client that uses a Thing directly. -Currently this is not a subclass of ThingClient, that may need to change. -It's a good idea to create a DirectThingClient at module level, so that type -hints work. +When `.Thing` objects interact on the server, it can be very useful to +use an interface that is identical to the `.ThingClient` used to access +the same `.Thing` remotely. This means that code can run either on the +server or on a client, e.g. in a Jupyter notebook where it is much +easier to debug. See :ref:`things_from_things` for more detail. + +Currently `.DirectThingClient` is not a subclass of `.ThingClient`, +that may need to change. It's a good idea to create a +`.DirectThingClient` at module level, so that type hints work. -This module may get moved in the near future. """ @@ -24,17 +29,45 @@ from fastapi import Request +__all__ = ["DirectThingClient", "direct_thing_client_class"] + + class DirectThingClient: + """A wrapper for `.Thing` that is a work-a-like for `.ThingClient`. + + This class is used to create a class that works like `.ThingClient` + but does not communicate over HTTP. Instead, it wraps a `.Thing` object + and calls its methods directly. + + It is not yet 100% identical to `.ThingClient`, in particular `.ThingClient` + returns a lot of data directly as deserialised from JSON, while this class + generally returns `pydantic.BaseModel` instances, without serialisation. + + `.DirectThingClient` is generally not used on its own, but is subclassed + (often dynamically) to add the actions and properties of a particular + `.Thing`. + """ + __globals__ = globals() # "bake in" globals so dependency injection works thing_class: type[Thing] + """The class of the underlying `.Thing` we are wrapping.""" thing_path: str + """The path to the Thing on the server. Relative to the server's base URL.""" def __init__(self, request: Request, **dependencies: Mapping[str, Any]): - """Wrapper for a Thing that makes it work like a ThingClient + r"""Wrap a `.Thing` so it works like a `.ThingClient`. - This class is designed to be used as a FastAPI dependency, and will retrieve a - Thing based on its `thing_path` attribute. Finding the Thing by class may also - be an option in the future. + This class is designed to be used as a FastAPI dependency, and will + retrieve a `.Thing` based on its ``thing_path`` attribute. + Finding the Thing by class may also be an option in the future. + + :param request: This is a FastAPI dependency to access the + `fastapi.Request` object, allowing access to various resources. + :param \**dependencies: Further arguments will be added + dynamically by subclasses, by duplicating this method and + manipulating its signature. Adding arguments with annotated + type hints instructs FastAPI to inject dependency arguments, + such as access to other `.Things`. """ server = find_thing_server(request.app) self._wrapped_thing = server.things[self.thing_path] @@ -50,10 +83,28 @@ def property_descriptor( writeable: bool = True, property_path: Optional[str] = None, ) -> PropertyClientDescriptor: - """Create a correctly-typed descriptor that gets and/or sets a property + """Create a correctly-typed descriptor that gets and/or sets a property. + + .. todo:: + This is copy-pasted from labthings_fastapi.client.__init__.property_descriptor + TODO: refactor this into a shared function. + + Create a descriptor object that wraps a property. This is for use on + a `.DirectThingClient` subclass. + + :param property_name: should be the name of the property (i.e. the + name it takes in the thing description, and also the name it is + assigned to in the class). + :param model: the Python ``type`` or a ``pydantic.BaseModel`` that + represents the datatype of the property. + :param description: text to use for a docstring. + :param readable: whether the property may be read (i.e. has ``__get__``). + :param writeable: whether the property may be written to. + :param property_path: the URL of the ``getproperty`` and ``setproperty`` + HTTP endpoints. Currently these must both be the same. These are + relative to the ``base_url``, i.e. the URL of the Thing Description. - This is copy-pasted from labthings_fastapi.client.__init__.property_descriptor - TODO: refactor this into a shared function. + :return: a descriptor allowing access to the specified property. """ class P(PropertyClientDescriptor): @@ -84,15 +135,60 @@ def __set__(self, obj: DirectThingClient, value: Any): return P() +class DependencyNameClashError(KeyError): + """A dependency argument name is used inconsistently. + + A current limitation of `.DirectThingClient` is that the dependency + arguments (see :ref:`dependencies`) are collected together in a single + dictionary. This makes the assumption that, if a name is reused, it is + reused for the same dependency. + + When names are re-used, we check if the values match. If not, this + exception is raised. + """ + + def __init__(self, name: str, existing: type, new: type): + """Create a DependencyNameClashError. + + See class docstring for an explanation of the error. + + :param name: the name of the clashing dependencies. + :param existing: the dependency type annotation in the dictionary. + :param new: the clashing type annotation. + """ + super().__init__( + f"{self.__doc__}\n\n" + f"This clash is with name: {name}.\n" + f"Its value is currently {existing}, which clashes with {new}." + ) + + def add_action( attrs: dict[str, Any], dependencies: list[inspect.Parameter], name: str, action: ActionDescriptor, ) -> None: - """Generates an action method and adds it to an attrs dict + """Generate an action method and adds it to an attrs dict. FastAPI Dependencies are appended to the `dependencies` list. + This list should later be converted to type hints on the class + initialiser, so that FastAPI supplies the dependencies when + the `.DirectThingClient` is initialised. + + :param attrs: the attributes of a soon-to-be-created `.DirectThingClient` + subclass. This will be passed to `type()` to create the subclass. + We will add the action method to this dictionary. + :param dependencies: lists the dependency parameters that will be + injected by FastAPI as arguments to the class ``__init__``. + Any dependency parameters of the supplied ``action`` should be + added to this list. + :param name: the name of the action. Should be the name of the + attribute, i.e. we will set ``attrs[name]``, and also match + the ``name`` in the supplied action descriptor. + :param action: an `.ActionDescriptor` to be wrapped. + + :raise DependencyNameClashError: if dependencies are inconsistent. """ @wraps(action.func) @@ -114,8 +210,8 @@ def action_method(self, **kwargs): # Currently, each name may only have one annotation, across # all actions - this is a limitation we should fix. if existing_param.annotation != param.annotation: - raise ValueError( - f"Conflicting dependency injection for {param.name}" + raise DependencyNameClashError( + param.name, existing_param.annotation, param.annotation ) included = True if not included: @@ -125,7 +221,18 @@ def action_method(self, **kwargs): def add_property( attrs: dict[str, Any], property_name: str, property: ThingProperty ) -> None: - """Add a property to a DirectThingClient subclass""" + """Add a property to a DirectThingClient subclass. + + We create a new descriptor using `.property_descriptor` and add it + to the ``attrs`` dictionary as ``property_name``. + + :param attrs: the attributes of a soon-to-be-created `.DirectThingClient` + subclass. This will be passed to `type()` to create the subclass. + We will add the property to this dictionary. + :param property_name: the name of the property. Should be the name of the + attribute, i.e. we will set ``attrs[name]``. + :param property: a `.PropertyDescriptor` to be wrapped. + """ attrs[property_name] = property_descriptor( property_name, property.model, @@ -139,10 +246,23 @@ def direct_thing_client_class( thing_class: type[Thing], thing_path: str, actions: Optional[list[str]] = None, -): - """Create a DirectThingClient from a Thing class and a path +) -> type[DirectThingClient]: + r"""Create a DirectThingClient from a Thing class and a path. This is a class, not an instance: it's designed to be a FastAPI dependency. + + :param thing_class: The `.Thing` subclass that will be wrapped. + :param thing_path: The path where the `.Thing` is found on the server. + :param actions: An optional list giving a subset of actions that will be + accessed. If this is specified, it may reduce the number of FastAPI + dependencies we need. + + :return: a subclass of `DirectThingClient` with attributes that match the + properties and actions of ``thing_class``. The ``__init__`` method + will have annotations that instruct FastAPI to supply all the + dependencies needed by its actions. + + This class may be used as a FastAPI dependency: see :ref:`things_from_things`. """ def init_proxy(self, request: Request, **dependencies: Mapping[str, Any]): diff --git a/src/labthings_fastapi/client/outputs.py b/src/labthings_fastapi/client/outputs.py index 47d4efef..061e9ca6 100644 --- a/src/labthings_fastapi/client/outputs.py +++ b/src/labthings_fastapi/client/outputs.py @@ -1,14 +1,33 @@ +"""A client-side implementation of `.Blob`. + +.. note:: + + In the future, both client and server code are planned to use `.Blob` to + represent binary data, or data held in a file. + +When a `.ThingClient` returns data to a client that matches the schema of a `.Blob` +(specifically, it needs an `href` and a `media_type`), we convert it into a +`.ClientBlobOutput` object. This is a work-a-like for `.Blob`, meaning it can +be saved to a file or have its contents accessed in the same ways. +""" + import io from typing import Optional import httpx class ClientBlobOutput: - """An output from LabThings best returned as a file + """An output from LabThings best returned as a file. This object is returned by a client when the output is not serialised to JSON. - It may be either retrieved to memory using `.content`, or saved to a file using - `.save()`. + It may be either retrieved to memory using `.ClientBlobOutput.content`, or + saved to a file using `.ClientBlobOutput.save`. + + .. note:: + + In the future, it is planned to replace this with `.Blob` as used on + server-side code. The ``.content`` and ``.save()`` methods should be + identical between the two. """ media_type: str @@ -17,13 +36,20 @@ class ClientBlobOutput: def __init__( self, media_type: str, href: str, client: Optional[httpx.Client] = None ): + """Create a ClientBlobOutput to wrap a link to a downloadable file. + + :param media_type: the MIME type of the remote file. + :param href: the URL where it may be downloaded. + :param client: if supplied, this `httpx.Client` will be used to + download the data. + """ self.media_type = media_type self.href = href self.client = client or httpx.Client() @property def content(self) -> bytes: - """Return the the output as a `bytes` object""" + """The binary data, as a `bytes` object.""" return self.client.get(self.href).content def save(self, filepath: str) -> None: @@ -31,10 +57,21 @@ def save(self, filepath: str) -> None: This may remove the need to hold the output in memory, though for now it simply retrieves the output into memory, then writes it to a file. + + :param filepath: the file will be saved at this location. """ with open(filepath, "wb") as f: f.write(self.content) def open(self) -> io.IOBase: - """Open the output as a binary file-like object.""" + """Open the output as a binary file-like object. + + Internally, this will download the file to memory, and wrap the + resulting `bytes` object in an `io.BytesIO` object to allow it to + function as a file-like object. + + To work with the data on disk, use `.ClientBlobOutput.save` instead. + + :return: a file-like object containing the downloaded data. + """ return io.BytesIO(self.content) diff --git a/src/labthings_fastapi/decorators/__init__.py b/src/labthings_fastapi/decorators/__init__.py index 8ae57991..32bf8307 100644 --- a/src/labthings_fastapi/decorators/__init__.py +++ b/src/labthings_fastapi/decorators/__init__.py @@ -1,7 +1,8 @@ -""" -The decorators in this module mark the Interaction Affordances of a Thing. +"""Mark the Interaction Affordances of a Thing. + +See :ref:`wot_cc` for definitions of Interaction Affordance and other terms. -LabThings generates a "Thing Description" to allow actions, properties, and +LabThings generates a :ref:`wot_td` to allow actions, properties, and events to be used by client code. The descriptions of each "interaction affordance" rely on docstrings and Python type hints to provide a full description of the parameters, so it's important that you use these @@ -13,27 +14,30 @@ signature: if you want to add descriptions or validators to individual arguments, you may use `pydantic.Field` to do this. -## Actions +Actions +------- You can add an Action to a Thing by declaring a method, decorated with -`@thing_action`. Parameters are not usually needed, but can be supplied to set +:deco:`.thing_action`. Parameters are not usually needed, but can be supplied to set various options. -## Properties +Properties +---------- As with Actions, Properties can be declared by decorating either a function, or -an attribute, with `@thing_property`. You can use the decorator either on +an attribute, with :deco:`.thing_property`. You can use the decorator either on a function (in which case that -function acts as the "getter" just like with Python's `@property` decorator). +function acts as the "getter" just like with Python's :deco`property` decorator). -## Events +Events +------ -Events are created by decorating attributes with `@thing_event`. Functions are +Events are created by decorating attributes with :deco:`.thing_event`. Functions are not supported at this time. """ from functools import wraps, partial -from typing import Optional, Callable +from typing import Optional, Callable, overload from ..descriptors import ( ActionDescriptor, ThingProperty, @@ -45,10 +49,16 @@ def mark_thing_action(func: Callable, **kwargs) -> ActionDescriptor: - """Mark a method of a Thing as an Action + r"""Mark a method of a Thing as an Action. + + We replace the function with a descriptor that's a + subclass of `.ActionDescriptor` + + :param func: The function to be decorated. + :param \**kwargs: Additional keyword arguments are passed to the constructor + of `.ActionDescriptor`. - We replace the function with a `Descriptor` that's a - subclass of `ActionDescriptor` + :return: An `.ActionDescriptor` wrapping the method. """ class ActionDescriptorSubclass(ActionDescriptor): @@ -57,8 +67,49 @@ class ActionDescriptorSubclass(ActionDescriptor): return ActionDescriptorSubclass(func, **kwargs) +@overload +def thing_action(func: Callable, **kwargs) -> ActionDescriptor: ... + + +@overload +def thing_action( + **kwargs, +) -> Callable[ + [ + Callable, + ], + ActionDescriptor, +]: ... + + @wraps(mark_thing_action) -def thing_action(func: Optional[Callable] = None, **kwargs): +def thing_action( + func: Optional[Callable] = None, **kwargs +) -> ( + ActionDescriptor + | Callable[ + [ + Callable, + ], + ActionDescriptor, + ] +): + r"""Mark a method of a `.Thing` as a LabThings Action. + + Methods decorated with :deco:`thing_action` will be available to call + over HTTP as actions. See :ref:`actions` for an introduction to the concept + of actions. + + This decorator may be used with or without arguments. + + :param func: The method to be decorated as an action. + :param \**kwargs: Keyword arguments are passed to the constructor + of `.ActionDescriptor`. + + :return: Whether used with or without argumnts, the result is that + the method is wrapped in an `.ActionDescriptor`, so it can be + called as usual, but will also be exposed over HTTP. + """ # This can be used with or without arguments. # If we're being used without arguments, we will # have a non-None value for `func` and defaults @@ -73,15 +124,23 @@ def thing_action(func: Optional[Callable] = None, **kwargs): def thing_property(func: Callable) -> ThingProperty: - """Mark a method of a Thing as a LabThings Property + """Mark a method of a Thing as a LabThings Property. This should be used as a decorator with a getter and a setter - just like a standard python property decorator. If extra functionality - is not required in the decorator, then using the ThingProperty class + just like a standard python `property` decorator. If extra functionality + is not required in the decorator, then using the `.ThingProperty` class directly may allow for clearer code - As properties are accessed over the HTTP API they need to be JSON serialisable - only return standard python types, or Pydantic BaseModels + Properties should always have a type annotation. This type annotation + will be used in automatic documentation and also to serialise the value + to JSON when it is sent over th network. This mean that the type of your + property should either be JSON serialisable (i.e. simple built-in types) + or a subclass of `pydantic.BaseModel`. + + :param func: A method to use as the getter for the new property. + + :return: A `.ThingProperty` descriptor that works like `property` but + allows the value to be read over HTTP. """ # Replace the function with a `Descriptor` that's a `ThingProperty` return ThingProperty( @@ -95,28 +154,38 @@ def thing_property(func: Callable) -> ThingProperty: def thing_setting(func: Callable) -> ThingSetting: """Mark a method of a Thing as a LabThings Setting. - A setting is a property that persists between runs. + A setting is a property that is saved to disk, so it persists even when + the LabThings server is restarted. This should be used as a decorator with a getter and a setter just like a standard python property decorator. If extra functionality - is not required in the decorator, then using the ThingSetting class - directly may allow for clearer code where the property works like a normal variable. + is not required in the decorator, then using the `ThingSetting` class + directly may allow for clearer code where the property works like a + variable. - When creating a Setting using this decorator you must always create a setter - as it is used to load the value from disk. + When creating a setting using this decorator, you must always add a setter + as it is used to load the value from disk. This follows the same syntax as + for `property`, i.e. a second function with the same name, decorated with + ``@my_property_name.setter``. - As settings are accessed over the HTTP API and saved to disk they need to be - JSON serialisable only return standard python types, or Pydantic BaseModels. + A type annotation is required, and should follow the same constraints as + for :deco:`thing_property`. If the type is a pydantic BaseModel, then the setter must also be able to accept the dictionary representation of this BaseModel as this is what will be used to set the Setting when loading from disk on starting the server. - Note: If a setting is mutated rather than set, this will not trigger saving. - For example: if a Thing has a setting called `dictsetting` holding the dictionary - `{"a": 1, "b": 2}` then `self.dictsetting = {"a": 2, "b": 2}` would trigger saving - but `self.dictsetting[a] = 2` would not, as the setter for `dictsetting` is never - called. + .. note:: + If a setting is mutated rather than set, this will not trigger saving. + For example: if a Thing has a setting called ``dictsetting`` holding the + dictionary ``{"a": 1, "b": 2}`` then ``self.dictsetting = {"a": 2, "b": 2}`` + would trigger saving but ``self.dictsetting[a] = 2`` would not, as the + setter for ``dictsetting`` is never called. + + :param func: A method to use as the getter for the new property. + + :return: A `.ThingSetting` descriptor that works like `property` but + allows the value to be read over HTTP and saves it to disk. """ # Replace the function with a `Descriptor` that's a `ThingSetting` return ThingSetting( @@ -127,10 +196,49 @@ def thing_setting(func: Callable) -> ThingSetting: ) -def fastapi_endpoint(method: HTTPMethod, path: Optional[str] = None, **kwargs): - """Add a function to FastAPI as an endpoint""" +def fastapi_endpoint( + method: HTTPMethod, path: Optional[str] = None, **kwargs +) -> Callable[[Callable], EndpointDescriptor]: + r"""Mark a function as a FastAPI endpoint without making it an action. + + This decorator will cause a method of a `.Thing` to be directly added to + the HTTP API, bypassing the machinery underlying Action and Property + affordances. Such endpoints will not be documented in the :ref:`wot_td` but + may be used as the target of links. For example, this could allow a file + to be downloaded from the `.Thing` at a known URL, or serve a video stream + that wouldn't be supported as a `.Blob`\ . + + The majority of `.Thing` implementations won't need this decorator, but + it is here to enable flexibility when it's needed. + + This decorator always takes arguments; in particular, ``method`` is + required. It should be used as: + + .. code-block:: python + + class DownloadThing(Thing): + @fastapi_endpoint("get") + def plain_text_response(self) -> str: + return "example string" + + This decorator is intended to work very similarly to the `fastapi` decorators + ``@app.get``, ``@app.post``, etc., with two changes: + + 1. The path is relative to the host `.Thing` and will default to the name + of the method. + 2. The method will be called with the host `.Thing` as its first argument, + i.e. it will be bound to the class as usua. + + :param method: The HTTP verb this endpoint responds to. + :param path: The path, relative to the host `.Thing` base URL. + :param \**kwargs: Additional keyword arguments are passed to the + `fastapi.FastAPI.get` decorator if ``method`` is ``get``, or to + the equivalent decorator for other HTTP verbs. + + :return: When used as intended, the result is an `.EndpointDescriptor`. + """ - def decorator(func): + def decorator(func: Callable) -> EndpointDescriptor: return EndpointDescriptor(func, http_method=method, path=path, **kwargs) return decorator diff --git a/src/labthings_fastapi/dependencies/__init__.py b/src/labthings_fastapi/dependencies/__init__.py index e69de29b..e3a66534 100644 --- a/src/labthings_fastapi/dependencies/__init__.py +++ b/src/labthings_fastapi/dependencies/__init__.py @@ -0,0 +1,15 @@ +r"""Resources that may be requested using annotated types. + +:ref:`actions` often need to access resources outside of the host `.Thing`\ , for +example invoking actions or accessing properties on other `.Thing`\ s or +calling methods provided by the server. + +:ref:`dependencies` are a `FastAPI concept`_ that is re-used in LabThings to allow +:ref:`actions` to request resources in a way that plays nicely with type hints +and is easy to intercept for testing. + +There is more documentation at :ref:`dependencies` for how this works within +LabThings. + +.. _`FastAPI concept`: https://fastapi.tiangolo.com/tutorial/dependencies/ +""" diff --git a/src/labthings_fastapi/dependencies/action_manager.py b/src/labthings_fastapi/dependencies/action_manager.py index 9b99a799..ada240e6 100644 --- a/src/labthings_fastapi/dependencies/action_manager.py +++ b/src/labthings_fastapi/dependencies/action_manager.py @@ -1,21 +1,25 @@ -""" -Context Var access to the Action Manager +"""Context Var access to the Action Manager. This module provides a context var to access the Action Manager instance. While LabThings tries pretty hard to conform to FastAPI's excellent convention that everything should be passed as a parameter, there are some cases where that's hard. In particular, generating URLs when responses are serialised is -difficult, because `pydantic` doesn't have a way to pass in extra context. +difficult, because `pydantic` doesn't have a way to access the `fastapi.Request` +object and use `fastapi.Request.url_for`. + +If an endpoint uses the `.ActionManagerDep` dependency, then the `.ActionManager` +is supplied as an argument. More usefully, when the output is serialised the +`.ActionManager` is available using `.ActionManagerContext.get()`. -If an endpoint uses the `ActionManagerDep` dependency, then the Action Manager -is available using `ActionManagerContext.get()`. +This is currently only used by `.Blob` objects, as "serialising" a `.Blob` +involves adding it to the `.ActionManager` and generating a download URL. """ from __future__ import annotations from contextvars import ContextVar -from typing import Annotated +from typing import Annotated, AsyncGenerator from typing_extensions import TypeAlias from fastapi import Depends, Request from ..dependencies.thing_server import find_thing_server @@ -23,14 +27,18 @@ def action_manager_from_thing_server(request: Request) -> ActionManager: - """Retrieve the Action Manager from the Thing Server + """Retrieve the Action Manager from the Thing Server. + + This is for use as a FastAPI dependency. We use the ``request`` to + access the `.ThingServer` and thus access the `.ActionManager`. - This is for use as a FastAPI dependency, so the thing server is - retrieved from the request object. + :param request: the FastAPI request object. This will be supplied by + FastAPI when this function is used as a dependency. + + :return: the `.ActionManager` object associated with our `.ThingServer`. """ action_manager = find_thing_server(request.app).action_manager - if action_manager is None: - raise RuntimeError("Could not get the blocking portal from the server.") + assert action_manager is not None return action_manager @@ -43,10 +51,19 @@ def action_manager_from_thing_server(request: Request) -> ActionManager: ActionManagerContext = ContextVar[ActionManager]("ActionManagerContext") -async def make_action_manager_context_available(action_manager: ActionManagerDep): - """Make the Action Manager available in the context +async def make_action_manager_context_available( + action_manager: ActionManagerDep, +) -> AsyncGenerator[ActionManager]: + """Make the Action Manager available in the context variable. + + The action manager may be accessed using `ActionManagerContext.get()` within + this context manager. + + :param action_manager: The `.ActionManager` object. Note that this is an + annotated type so it will be supplied automatically when used as a FastAPI + dependency. - The action manager may be accessed using `ActionManagerContext.get()`. + :yield: the `.ActionManager` object. """ ActionManagerContext.set(action_manager) yield action_manager diff --git a/src/labthings_fastapi/dependencies/blocking_portal.py b/src/labthings_fastapi/dependencies/blocking_portal.py index 41b3ac85..bbc4bb05 100644 --- a/src/labthings_fastapi/dependencies/blocking_portal.py +++ b/src/labthings_fastapi/dependencies/blocking_portal.py @@ -1,7 +1,26 @@ -"""FastAPI dependency for a blocking portal +"""FastAPI dependency for a blocking portal. This allows dependencies that are called by threaded code to send things back -to the async event loop. +to the async event loop. See :ref:`concurrency` for more details. + +Threaded code can call asynchronous code in the `anyio` event loop used by +`fastapi`, if an `anyio.BlockingPortal` is used. + +The `.ThingServer` sets up an `anyio.from_thread.BlockingPortal` when the server starts +(in `.ThingServer.lifespan`). This may be accessed from an action using the +`.BlockingPortal` dependency in this module. + +.. note:: + + The blocking portal is accessed via a dependency to ensure we only ever + use the blocking portal attached to the server handling the current + request. + + This may be simplified in the future, as a `.Thing` can only ever be + attached to one `.ThingServer`, and each `.ThingServer` corresponds + to exactly one event loop. That means a mechanism may be introduced in + the future that allows `.Thing` code to access a blocking portal without + the need for a dependency. """ from __future__ import annotations @@ -12,14 +31,23 @@ def blocking_portal_from_thing_server(request: Request) -> RealBlockingPortal: - """Return the blocking portal from our ThingServer + r"""Return the blocking portal from our ThingServer. This is for use as a FastAPI dependency, to allow threaded code to call - async code. + async code. See the module-level docstring for :mod:`.blocking_portal`. + + :param request: The `fastapi.Request` object, supplied by the :ref:`dependencies` + mechanism. + + :return: the `anyio.from_thread.BlockingPortal` allowing access to te + `.ThingServer`\ 's event loop. """ portal = find_thing_server(request.app).blocking_portal - if portal is None: - raise RuntimeError("Could not get the blocking portal from the server.") + assert portal is not None, RuntimeError( + "Could not get the blocking portal from the server." + # This should never happen, as the blocking portal is added + # and removed in `.ThingServer.lifecycle`. + ) return portal @@ -28,7 +56,7 @@ def blocking_portal_from_thing_server(request: Request) -> RealBlockingPortal: ] """ A ready-made dependency type for a blocking portal. If you use an argument with -type `BlockingPortal`, FastAPI will automatically inject the blocking portal. -This is simply shorthand for {class}`anyio.from_thread.BlockingPortal` annotated with -`Depends(blocking_portal_from_thing_server)`. +type `.BlockingPortal`, FastAPI will automatically inject the blocking portal. +This is simply shorthand for `anyio.from_thread.BlockingPortal` annotated with +``Depends(blocking_portal_from_thing_server)``. """ diff --git a/src/labthings_fastapi/dependencies/invocation.py b/src/labthings_fastapi/dependencies/invocation.py index 247b2926..000d55af 100644 --- a/src/labthings_fastapi/dependencies/invocation.py +++ b/src/labthings_fastapi/dependencies/invocation.py @@ -1,4 +1,35 @@ -"""FastAPI dependency for an invocation ID""" +"""FastAPI dependencies for invocation-specific resources. + +There are a number of LabThings-FastAPI features that are specific to each +invocation of an action. These may be accessed using the :ref:`dependencies` in +this module. + +It's important to understand how FastAPI handles dependencies when looking +at the code in this module. Each dependency (i.e. each callable passed as +the argument to `fastapi.Depends` in an annotated type) will be evaluated +only once per HTTP request. This means that we don't need to cache +`.InvocationID` and pass it between the functions, because the same ID +will be passed to every dependency that has an argument with the annotated +type `.InvocationID`. + +When an action is invoked with a ``POST`` request, the endpoint function +responsible always has dependencies for the `.InvocationID` and +`.CancelHook`. These are added to the `.Invocation` thread that is created. +If the action declares dependencies with these types, it will receive the +same objects. This avoids the need for the action to be aware of its +`.Invocation`. + +.. note:: + + Currently, `.invocation_logger` is called from `.Invocation.run` with the + invocation ID as an argument, and is not a direct dependency of the action's + ``POST`` endpoint. + + This doesn't duplicate the returned logger object, as + `logging.getLogger` may be called multiple + times and will return the same `logging.Logger` object provided it is + called with the same name. +""" from __future__ import annotations import uuid @@ -9,21 +40,52 @@ def invocation_id() -> uuid.UUID: - """Return a UUID for an action invocation + """Generate a UUID for an action invocation. - This is for use as a FastAPI dependency, to allow other dependencies to - access the invocation ID. Useful for e.g. file management. + This is for use as a FastAPI dependency (see :ref:`dependencies`). + + Because `fastapi` only evaluates each dependency once per HTTP + request, the `.UUID` we generate here is available to all of + the dependencies declared by the ``POST`` endpoint that starts + an action. + + Any dependency that has a parameter with the type hint + `.InvocationID` will be supplied with the ID we generate + here, it will be consistent within one HTTP request, and will + be unique for each request (i.e. for each invocation of the + action). + + This dependency is used by the `.InvocationLogger`, `.CancelHook` + and other resources to ensure they all have the same ID, even + before the `.Invocation` object has been created. + + :return: A unique ID for the current HTTP request, i.e. for this + invocation of an action. """ return uuid.uuid4() InvocationID = Annotated[uuid.UUID, Depends(invocation_id)] +"""A FastAPI dependency that supplies the invocation ID. + +This calls :func:`.invocation_id` to generate a new `.UUID`. It is used +to supply the invocation ID when an action is invoked. + +Any dependency of an action may access the invocation ID by +using this dependency. +""" def invocation_logger(id: InvocationID) -> logging.Logger: - """Retrieve a logger object for an action invocation + """Make a logger object for an action invocation. + + This function should be used as a dependency for an action, and + will supply a logger that's specific to each invocation of that + action. This is how `.Invocation.log` is generated. - This will have a level of at least INFO. + :param id: The Invocation ID, supplied as a FastAPI dependency. + + :return: A `logging.Logger` object specific to this invocation. """ logger = logging.getLogger(f"labthings_fastapi.actions.{id}") logger.setLevel(logging.INFO) @@ -31,6 +93,11 @@ def invocation_logger(id: InvocationID) -> logging.Logger: InvocationLogger = Annotated[logging.Logger, Depends(invocation_logger)] +"""A FastAPI dependency supplying a logger for the action invocation. + +This calls `.invocation_logger` to generate a logger for the current +invocation. For details of how to use dependencies, see :ref:`dependencies`. +""" class InvocationCancelledError(BaseException): @@ -38,28 +105,83 @@ class InvocationCancelledError(BaseException): Note that this inherits from BaseException so won't be caught by `except Exception`, it must be handled specifically. + + Action code may want to handle cancellation gracefully. This + exception should be propagated if the action's status should be + reported as ``cancelled``, or it may be handled so that the + action finishes, returns a value, and is marked as ``completed``. + + If this exception is handled, the `.CancelEvent` should be reset + to allow another `.InvocationCancelledError` to be raised if the + invocation receives a second cancellation signal. """ class CancelEvent(threading.Event): + """An Event subclass that enables cancellation of actions. + + This `threading.Event` subclass adds methods to raise + `.InvocationCancelledError` exceptions if the invocation is cancelled, + usually by a ``DELETE`` request to the invocation's URL. + """ + def __init__(self, id: InvocationID): + """Initialise the cancellation event. + + :param id: The invocation ID, annotated as a dependency so it is + supplied automatically by FastAPI. + """ threading.Event.__init__(self) self.invocation_id = id def raise_if_set(self): - """Raise a CancelledError if the event is set""" + """Raise an exception if the event is set. + + This is intended as a compact alternative to: + + .. code-block:: + + if cancel_event.is_set(): + raise InvocationCancelledError() + + :raise InvocationCancelledError: if the event has been cancelled. + """ if self.is_set(): raise InvocationCancelledError("The action was cancelled.") def sleep(self, timeout: float): - """Sleep for a given time in seconds, but raise an exception if cancelled""" + r"""Sleep for a given time in seconds, but raise an exception if cancelled. + + This function can be used in place of `time.sleep`. It will usually behave + the same as `time.sleep`\ , but if the cancel event is set during the time + when we are sleeping, an exception is raised to interrupt the sleep and + cancel the action. + + :param timeout: The time to sleep for, in seconds. + + :raise InvocationCancelledError: if the event has been cancelled. + """ if self.wait(timeout): raise InvocationCancelledError("The action was cancelled.") def invocation_cancel_hook(id: InvocationID) -> CancelHook: - """Get a cancel hook belonging to a particular invocation""" + """Make a cancel hook for a particular invocation. + + This is for use as a FastAPI dependency, and will create a + `.CancelEvent` for use with a particular `.Invocation`. + + :param id: The invocation ID, supplied by FastAPI. + + :return: a `.CancelHook` event. + """ return CancelEvent(id) CancelHook = Annotated[CancelEvent, Depends(invocation_cancel_hook)] +"""FastAPI dependency for an event that allows invocations to be cancelled. + +This is an annotated type that returns a `.CancelEvent`, which can be used +to raise `.InvocationCancelledError` exceptions if the invocation is +cancelled, usually by a ``DELETE`` request to the invocation's URL. +""" diff --git a/src/labthings_fastapi/dependencies/metadata.py b/src/labthings_fastapi/dependencies/metadata.py index 9fe55c50..5f87d732 100644 --- a/src/labthings_fastapi/dependencies/metadata.py +++ b/src/labthings_fastapi/dependencies/metadata.py @@ -1,3 +1,10 @@ +"""FastAPI dependency to get metadata from all Things. + +This module defines a FastAPI dependency (see :ref:`dependencies`) that will +retrieve metadata from every `.Thing` on the server. This is intended to +simplify the task of adding metadata to data collected by `.Thing` instances. +""" + from __future__ import annotations from typing import Annotated, Any, Callable from collections.abc import Mapping @@ -8,23 +15,47 @@ def thing_states_getter(request: Request) -> Callable[[], Mapping[str, Any]]: - """A dependency to retrieve summary metadata from all Things in this server. + """Generate a function to retrieve metadata from all Things in this server. + + This is intended to make it easy for a `.Thing` to summarise the other + `.Things` in the same server, as is often appropriate when embedding metadata + in data files. For example, it's used to populate the ``UserComment`` + EXIF field in images saved by the OpenFlexure Microscope. + + This is intended for use as a FastAPI dependency, so the ``request`` argument + will be supplied automatically. + + This function does not collect the metadata when it is run. Instead, we + return a function that will collect the metadata when it is called. This + delays collection of metadata until it is needed. - This is intended to make it easy for a Thing to summarise the other Things - it's associated with, for example it's used to populate the UserComment - EXIF field in the OpenFlexure Microscope. + Delaying collection of metadata is useful because FastAPI dependencies are + evaluated only once, before the action starts. If we collect metadata then, + there is no way for it to change during an action, so the metadata may be + out of date. - `thing_states_getter` differs from `get_thing_states` because the latter - is evaluated once, before the action, and this dependency returns a function - that collects the metadata when it's run. + For example, if we take + a Z stack of microscope images, we need to collect metadata after each image + in order to ensure the recorded position of the stage is up to date. - If your action is likely to be run from other actions where the metadata - changes, you should use this version. + Bear in mind that actions may call other actions, so even if you have + a very simple or short action that will not cause metadata to change, it + may be called by a longer action where that isn't true. Dependencies will be + evaluated before the calling action starts, so stale metadata is still a + possibility in very short actions. + + :param request: the `fastapi.Request` object, supplied automatically when + used as a dependency. See :ref:`dependencies`. + + :return: a function that returns a dictionary of metadata. """ server = find_thing_server(request.app) - def get_metadata(): - """Retrieve metadata from each Thing""" + def get_metadata() -> dict[str, Any]: + """Retrieve metadata from all Things on the server. + + :return: a dictionary of metadata, with the `.Thing` names as keys. + """ return {k: v.thing_state for k, v in server.things.items()} return get_metadata @@ -33,3 +64,12 @@ def get_metadata(): GetThingStates = Annotated[ Callable[[], Mapping[str, Any]], Depends(thing_states_getter) ] +"""A ready-made FastAPI dependency, returning a function to collect metadata. + +This calls `.thing_states_getter` to provide a function that supplies a +dictionary of metadata. It describes the state of all `.Thing` instances on +the current `.ThingServer` as reported by their ``thing_state`` property. + +Use this wherever you need to collect summary metadata to embed in data +files. +""" diff --git a/src/labthings_fastapi/dependencies/raw_thing.py b/src/labthings_fastapi/dependencies/raw_thing.py index 026ae382..19858689 100644 --- a/src/labthings_fastapi/dependencies/raw_thing.py +++ b/src/labthings_fastapi/dependencies/raw_thing.py @@ -1,3 +1,10 @@ +"""FastAPI dependency to obtain a `.Thing` directly. + +This module allows actions to obtain a `.Thing` instance without the +`.DirectThingClient` wrapper. As a rule, it is best to use `.DirectThingClient` +where possible. +""" + from __future__ import annotations from typing import Annotated, Callable, TypeVar @@ -13,33 +20,50 @@ def find_raw_thing_by_class( cls: type[ThingInstance], ) -> Callable[[Request], ThingInstance]: - """Generate a function that locates the instance of a specific Thing subclass + """Generate a function that locates the instance of a Thing subclass. + + .. warning:: - ```{warning} - Using a raw {class}`..thing.Thing` can be tricky: unless you really need to, it is - usually better to use an internal thing client, which provides an interface that - should be identical to the HTTP thing client in Python. This is safer, and means - code should be easier to translate between server and client-side. - ``` + Using a `.Thing` directly can be tricky: unless you really need to, it is + usually better to use a `.DirectThingClient`, which provides an interface + that should be identical to the HTTP thing client in Python. This is safer, + and means code should be easier to translate between server and client-side. - In order to access the instance of `OtherThing` attached to your thing server, + + In order to access the instance of ``OtherThing`` attached to your thing server, declare your argument type as: - ```{code-block} python - def endpoint( - other_thing: Annotated[OtherThing, Depends(find_thing_by_class(OtherThing)] - ): - pass - ``` + .. code-block:: python + + OtherThingDep = Annotated[ + OtherThing, Depends(find_raw_thing_by_class(OtherThing)) + ] - LabThings will supply this argument automatically through the FastAPI dependency + + def endpoint(other_thing: OtherThingDep): + pass + + LabThings will supply this argument automatically through the :ref:`dependencies` mechanism. + + Note that this function *returns* a dependency - it should be called with + arguments inside `fastapi.Depends`. + + :param cls: is the `.Thing` subclass that will be returned by the dependency. + + :return: a dependency suitable for use with `fastapi.Depends` (see example). """ def find_raw_thing(request: Request) -> ThingInstance: - """A dependency to locate a Thing based on its class. + """Locate a Thing based on its class. + + This function is generated by `.find_raw_thing_by_class`, see + the documentation there. + + :param request: is supplied by FastAPI - This function is generated by :func:`find_raw_thing_by_class`, see that for docs. + :return: an instance of the `.Thing` subclass specified when the + dependency was created. """ server = find_thing_server(request.app) return server.thing_by_class(cls) @@ -50,34 +74,44 @@ def find_raw_thing(request: Request) -> ThingInstance: def raw_thing_dependency(cls: type[ThingInstance]) -> type[ThingInstance]: """Generate a dependency that will supply a particular Thing at runtime. - ```{warning} - Using a raw {class}`labthings_fastapi.thing.Thing` can be tricky: unless you really need to, it is - usually better to use {class}`labthings_fastapi.dependencies.thing.direct_thing_client_dependency`, - which provides an interface that should be identical to the HTTP thing - client in Python. This is safer, and means code should be easier to - translate between server and client-side. - ``` + .. warning:: + + If it is possible to use a `.direct_thing_client_dependency` instead, + that is preferable. The current function supplies a `.Thing` directly + and does not supply dependency parameters or enforce the public API. + + This function should make it possible for an action to obtain a `.Thing` + object directly. If you declare a type alias using this function, it will + include an annotation that prompts FastAPI to supply the instance of the + class. - This function should make it simple to depend on a {class}`Thing` object directly. If - you declare a type alias using this function, it will include an annotation that - prompts FastAPI to supply the instance of the class. + .. warning:: + + Most linters and type checkers will not accept the result of a function + call as a valid type. It may be preferable to use + `.find_raw_thing_by_class` directly, even though it is slightly more + verbose. Usage: - ```{code-block} python + .. code-block:: python + + from my_other_thing import MyOtherThing as MyOtherThingClass - from my_other_thing import MyOtherThing as MyOtherThingClass + MyOtherThing = raw_thing_dependency(MyOtherThingClass) - MyOtherThing = raw_thing_dependency(MyOtherThingClass) - class MyThing(Thing): - @thing_action - def do_something(self, other_thing: MyOtherThing) -> None: - "This action needs no arguments" - other_thing.function_only_available_in_python() - ``` + class MyThing(Thing): + @thing_action + def do_something(self, other_thing: MyOtherThing) -> None: + "This action needs no arguments" + other_thing.function_only_available_in_python() + :param cls: The class of the Thing that will be supplied - :return: A type alias that works as a dependency to supply an instance of ``cls`` - at runtime. + + :return: An annotated type that works as a dependency to supply an + instance of ``cls`` at runtime. """ - return Annotated[cls, Depends(find_raw_thing_by_class(cls))] # type: ignore[return-value] + return Annotated[ # type: ignore[return-value] + cls, Depends(find_raw_thing_by_class(cls)) + ] diff --git a/src/labthings_fastapi/dependencies/thing.py b/src/labthings_fastapi/dependencies/thing.py index 5e4968f3..9b7abf87 100644 --- a/src/labthings_fastapi/dependencies/thing.py +++ b/src/labthings_fastapi/dependencies/thing.py @@ -1,3 +1,17 @@ +r"""FastAPI dependency to allow `.Thing`\ s to depend on each other. + +This module defines a mechanism to obtain a `.DirectThingClient` that +wraps another `.Thing` on the same server. See :ref:`things_from_things` and +:ref:`dependencies` for more detail. + +.. note:: + + `.direct_thing_client_dependency` may confuse linters and type + checkers, as types should not be the result of a function call. + You may wish to manually create an annotated type using + `.direct_thing_client_class`. +""" + from __future__ import annotations from typing import Annotated, Optional @@ -12,7 +26,22 @@ def direct_thing_client_dependency( thing_path: str, actions: Optional[list[str]] = None, ) -> type[Thing]: - """A type annotation that causes FastAPI to supply a direct thing client + """Make an annotated type to allow Things to depend on each other. + + This function returns an annotated type that may be used as a FastAPI + dependency. The dependency will return a `.DirectThingClient` that + wraps the specified `.Thing`. This should be a drop-in replacement for + `.ThingClient` so that code is consistent whether run in an action, or + in a script or notebook on a remote computer. + + See :ref:`things_from_things` and :ref:`dependencies`. + + .. note:: + + `.direct_thing_client_dependency` may confuse linters and type + checkers, as types should not be the result of a function call. + You may wish to manually create an annotated type using + `.direct_thing_client_class`. :param thing_class: The class of the thing to connect to :param thing_path: The path to the thing on the server diff --git a/src/labthings_fastapi/dependencies/thing_server.py b/src/labthings_fastapi/dependencies/thing_server.py index ab9181f6..8689fc8e 100644 --- a/src/labthings_fastapi/dependencies/thing_server.py +++ b/src/labthings_fastapi/dependencies/thing_server.py @@ -1,9 +1,29 @@ -""" -Retrieve the ThingServer object +"""Retrieve the `.ThingServer` object. + +This module provides a function that will retrieve the `.ThingServer` +based on the `fastapi.Request` object. It may be used as a dependency with +``Annotated[ThingServer, Depends(thing_server_from_request)]``. + +See :ref:`dependencies` for more information on the dependency mechanism, +and :ref:`things_from_things` for more on how `.Things` interact. + +.. note:: + + This module does not provide a ready-made annotated type to use as a + dependency. Doing so would mean this module has a hard dependency on + `.ThingServer` and cause circular references. See above for the + annotated type, which you may define in any code that needs it. -This module provides a function that will retrieve the ThingServer -based on the `Request` object. It may be used as a dependency with: -`Annotated[ThingServer, Depends(thing_server_from_request)]`. +.. note:: + + The rationale for this function is that we want to make sure `.Thing` + instances only access the server associated with the current request. + This means that we use the `fastapi.Request` to look up the + `fastapi.FastAPI` app, and then use the app to look up the `.ThingServer`. + + As each `.Thing` is connected to exactly one `.ThingServer`, this may + become unnecessary in the future as the server could be exposed as a + property of the `.Thing`. """ from __future__ import annotations @@ -18,7 +38,26 @@ def find_thing_server(app: FastAPI) -> ThingServer: - """Find the ThingServer associated with an app""" + """Find the ThingServer associated with an app. + + This function will return the `.ThingServer` object that contains + a particular `fastapi.FastAPI` app. The app is available as part + of the `fastapi.Request` object, so this makes it possible to + get the `.ThingServer` in dependency functions. + + This function will not work as a dependency, but + `.thing_server_from_request` will. + + :param app: The `fastapi.FastAPI` application that implements the + `.ThingServer`, i.e. this is ``thing_server.app``. + + :return: the `.ThingServer` that owns the ``app``. + + :raise RuntimeError: if there is no `.ThingServer` associated + with the current FastAPI application. This should only happen + if this function is called on a `fastapi.FastAPI` instance + that was not created by a `.ThingServer`. + """ for server in _thing_servers: if server.app == app: return server @@ -26,9 +65,24 @@ def find_thing_server(app: FastAPI) -> ThingServer: def thing_server_from_request(request: Request) -> ThingServer: - """Retrieve the Action Manager from the Thing Server + """Retrieve the `.ThingServer` from a request. This is for use as a FastAPI dependency, so the thing server is - retrieved from the request object. + retrieved from the request object. See `.find_thing_server`. + + It may be used as a dependency with: + + .. code-block:: python + + ServerDep = Annotated[ThingServer, Depends(thing_server_from_request)] + + This is not provided as a ready-made annotated type because it would + introduce a hard dependency on the :mod:`.server` module and cause circular + references. + + :param request: is supplied automatically by FastAPI when used + as a dependency. + + :return: the `.ThingServer` handling the current request. """ return find_thing_server(request.app) diff --git a/src/labthings_fastapi/deps.py b/src/labthings_fastapi/deps.py index cec40edc..2b540ce9 100644 --- a/src/labthings_fastapi/deps.py +++ b/src/labthings_fastapi/deps.py @@ -1,11 +1,10 @@ -""" -FastAPI dependencies for LabThings. +"""FastAPI dependencies for LabThings. The symbols in this module are type annotations that can be used in the arguments of Action methods (or FastAPI endpoints) to automatically supply the required dependencies. -See the documentation on dependencies for more details of how to use +See the documentation on :ref:`dependencies` for more details of how to use these. """ @@ -14,6 +13,7 @@ from .dependencies.metadata import GetThingStates from .dependencies.raw_thing import raw_thing_dependency from .dependencies.thing import direct_thing_client_dependency +from .client.in_server import direct_thing_client_class, DirectThingClient # The symbols in __all__ are part of our public API. See note # in src/labthings_fastapi/__init__.py for more details. @@ -25,4 +25,6 @@ "GetThingStates", "raw_thing_dependency", "direct_thing_client_dependency", + "direct_thing_client_class", + "DirectThingClient", ] diff --git a/src/labthings_fastapi/descriptors/__init__.py b/src/labthings_fastapi/descriptors/__init__.py index 901b14ad..404cce21 100644 --- a/src/labthings_fastapi/descriptors/__init__.py +++ b/src/labthings_fastapi/descriptors/__init__.py @@ -1,5 +1,18 @@ -from .action import ActionDescriptor as ActionDescriptor -from .property import ThingProperty as ThingProperty -from .property import ThingSetting as ThingSetting -from .endpoint import EndpointDescriptor as EndpointDescriptor -from .endpoint import HTTPMethod as HTTPMethod +"""Descriptors to add :ref:`wot_affordances` to `.Thing` subclasses. + +This module will likely be removed in the next release. +""" + +from .action import ActionDescriptor +from .property import ThingProperty +from .property import ThingSetting +from .endpoint import EndpointDescriptor +from .endpoint import HTTPMethod + +__all__ = [ + "ActionDescriptor", + "ThingProperty", + "ThingSetting", + "EndpointDescriptor", + "HTTPMethod", +] diff --git a/src/labthings_fastapi/descriptors/action.py b/src/labthings_fastapi/descriptors/action.py index d2bf1084..58ed321c 100644 --- a/src/labthings_fastapi/descriptors/action.py +++ b/src/labthings_fastapi/descriptors/action.py @@ -1,11 +1,18 @@ -""" -Define an object to represent an Action, as a descriptor. -""" +"""Define an object to represent an Action, as a descriptor.""" from __future__ import annotations from functools import partial import inspect -from typing import TYPE_CHECKING, Annotated, Any, Callable, Optional, Literal, overload +from typing import ( + TYPE_CHECKING, + Annotated, + Any, + Callable, + Optional, + Literal, + Union, + overload, +) from weakref import WeakSet from fastapi import Body, FastAPI, Request, BackgroundTasks @@ -25,7 +32,7 @@ ) from ..outputs.blob import BlobIOContextDep from ..thing_description import type_to_dataschema -from ..thing_description.model import ActionAffordance, ActionOp, Form, Union +from ..thing_description._model import ActionAffordance, ActionOp, Form from ..utilities import labthings_data, get_blocking_portal from ..exceptions import NotConnectedToServerError @@ -37,8 +44,8 @@ ## Important note This `POST` request starts an Action, i.e. the server will do something -that may continue after the HTTP request has been responded to. The -response will always be an ActionInvocation object, that details the current +that may continue after the HTTP request has been responded to. The +response will always be an ActionInvocation object, that details the current status of the action and provides an interface to poll for completion. If the action completes within a specified timeout, we will return @@ -49,20 +56,53 @@ """ ACTION_GET_DESCRIPTION = """ -This will include times and input values, as well as output values for -actions that have completed. These actions will also show up under the -`action_invocations` endpoint, and can also be retrieved individually +This will include times and input values, as well as output values for +actions that have completed. These actions will also show up under the +`action_invocations` endpoint, and can also be retrieved individually using the link included in each action. """ class ActionDescriptor: + """Wrap actions to enable them to be run over HTTP. + + This class is responsible for generating the action description for + the :ref:`wot_td` and creating the function that responds to ``POST`` + requests to invoke the action. + + .. note:: + Descriptors are instantiated once per class. This means that we cannot + assume there is only one action corresponding to this descriptor: there + may be multiple `.Thing` instances with the same descriptor. That is + why the host `.Thing` must be passed to many functions as an argument, + and why observers, for example, must be keyed by the `.Thing` rather + than kept as a property of ``self``. + """ + def __init__( self, func: Callable, response_timeout: float = 1, retention_time: float = 300, ): + """Create a new action descriptor. + + The action descriptor wraps a method of a `.Thing`. It may still be + called from Python in the same way, but it will also be added to the + HTTP API and automatic documentation. + + :param func: is the method that will be run when the action is called. + :param response_timeout: is how long we should wait before returning a + response to the client. This is not currently used, as we always + return immediately with a `201` code. In the future, it may set a + default time to wait before responding. If the action finishes + before we respond, we will be able to return the completed action + and its output. If the action is still running, we return a 201 + code and data enabling the client to poll to find out the status + of the action. + :param retention_time: how long, in seconds, the action should be kept + for after it has completed. + """ self.func = func self.response_timeout = response_timeout self.retention_time = retention_time @@ -82,15 +122,17 @@ def __init__( self.invocation_model.__name__ = f"{self.name}_invocation" @overload - def __get__(self, obj: Literal[None], type=None) -> ActionDescriptor: ... + def __get__(self, obj: Literal[None], type=None) -> ActionDescriptor: # noqa: D105 + ... @overload - def __get__(self, obj: Thing, type=None) -> Callable: ... + def __get__(self, obj: Thing, type=None) -> Callable: # noqa: D105 + ... def __get__( - self, obj: Optional[Thing], type=None + self, obj: Optional[Thing], type: Optional[type[Thing]] = None ) -> Union[ActionDescriptor, Callable]: - """The function, bound to an object as for a normal method. + """Return the function, bound to an object as for a normal method. This currently doesn't validate the arguments, though it may do so in future. In its present form, this is equivalent to a regular @@ -98,6 +140,15 @@ def __get__( If `obj` is None, the descriptor is returned, so we can get the descriptor conveniently as an attribute of the class. + + :param obj: the `.Thing` to which we are attached. This will be + the first argument supplied to the function wrapped by this + descriptor. + :param type: the class of the `.Thing` to which we are attached. + If the descriptor is accessed via the class it is returned + directly. + :return: the action function, bound to ``obj`` (when accessed + via an instance), or the descriptor (accessed via the class). """ if obj is None: return self @@ -108,36 +159,50 @@ def __get__( @property def name(self): - """The name of the wrapped function""" + """The name of the wrapped function.""" return self.func.__name__ @property def title(self): - """A human-readable title""" + """A human-readable title.""" return get_summary(self.func) or self.name @property def description(self): - """A description of the action""" + """A description of the action.""" return get_docstring(self.func, remove_summary=True) - def _observers_set(self, obj): - """A set used to notify changes""" + def _observers_set(self, obj: Thing) -> WeakSet: + """Return a set used to notify changes. + + Note that we need to supply the `.Thing` we are looking at, as in + general there may be more than one object of the same type, and + descriptor instances are shared between all instances of their class. + + :param obj: The `.Thing` on which the action is being observed. + + :return: a weak set of callables to notify on changes to the action. + This is used by websocket endpoints. + """ ld = labthings_data(obj) if self.name not in ld.action_observers: ld.action_observers[self.name] = WeakSet() return ld.action_observers[self.name] - def emit_changed_event(self, obj, status): - """Notify subscribers that the action status has changed + def emit_changed_event(self, obj: Thing, status: str): + """Notify subscribers that the action status has changed. - This function is run from within the `Invocation` thread that - is created when an action is called. It must be run from this thread + This function is run from within the `.Invocation` thread that + is created when an action is called. It must be run from a thread as it is communicating with the event loop via an `asyncio` blocking - portal. + portal. Async code must not use the blocking portal as it can deadlock + the event loop. + + :param obj: The `.Thing` on which the action is being observed. + :param status: The status of the action, to be sent to observers. - :raises NotConnectedToServerError: if the Thing calling the action is not - connected to a server with a running event loop. + :raise NotConnectedToServerError: if the Thing calling the action is not + connected to a server with a running event loop. """ try: runner = get_blocking_portal(obj) @@ -154,11 +219,22 @@ def emit_changed_event(self, obj, status): status, ) except Exception: - # TODO: in the unit test, the get_blockint_port throws exception + # TODO: in the unit test, the get_blocking_portal throws exception ... - async def emit_changed_event_async(self, obj: Thing, value: Any): - """Notify subscribers that the action status has changed""" + async def emit_changed_event_async(self, obj: Thing, value: Any) -> None: + """Notify subscribers that the action status has changed. + + This is an async function that must be run in the `anyio` event loop. + It will send messages to each observer to notify them that something + has changed. + + :param obj: The `.Thing` on which the action is defined. + `.ActionDescriptor` objects are unique to the class, but there may + be more than one `.Thing` attached to a server with the same class. + We use ``obj`` to look up the observers of the current `.Thing`. + :param value: The action status to communicate to the observers. + """ action_name = self.name for observer in self._observers_set(obj): await observer.send( @@ -168,8 +244,18 @@ async def emit_changed_event_async(self, obj: Thing, value: Any): } ) - def add_to_fastapi(self, app: FastAPI, thing: Thing): - """Add this action to a FastAPI app, bound to a particular Thing.""" + def add_to_fastapi(self, app: FastAPI, thing: Thing) -> None: + """Add this action to a FastAPI app, bound to a particular Thing. + + This function creates two functions to handle ``GET`` and ``POST`` + requests to the action's endpoint, and adds them to the `fastapi.FastAPI` + aplication. + + :param app: The `fastapi.FastAPI` app to add the endpoint to. + :param thing: The `.Thing` to which the action is attached. Bear in + mind that the descriptor may be used by more than one `.Thing`, + so this can't be a property of the descriptor. + """ # We can't use the decorator in the usual way, because we'd need to # annotate the type of `body` with `self.model` which is only defined @@ -254,12 +340,22 @@ def start_action( summary=f"All invocations of {self.name}.", ) def list_invocations(action_manager: ActionManagerContextDep): - return action_manager.list_invocations(self, thing, as_responses=True) + return action_manager.list_invocations(self, thing) def action_affordance( self, thing: Thing, path: Optional[str] = None ) -> ActionAffordance: - """Represent the property in a Thing Description.""" + """Represent the property in a Thing Description. + + This function describes the Action in :ref:`wot_td` format. + + :param thing: The `.Thing` to which the action is attached. + :param path: The prefix applied to all endpoints associated with the + `.Thing`. This is the URL for the Thing Description. If it is + omitted, we use the ``path`` property of the ``thing``. + + :return: An `.ActionAffordance` describing this action. + """ path = path or thing.path forms = [ Form[ActionOp](href=path + self.name, op=[ActionOp.invokeaction]), diff --git a/src/labthings_fastapi/descriptors/endpoint.py b/src/labthings_fastapi/descriptors/endpoint.py index 4892f259..6beec6a1 100644 --- a/src/labthings_fastapi/descriptors/endpoint.py +++ b/src/labthings_fastapi/descriptors/endpoint.py @@ -1,3 +1,18 @@ +"""Add a FastAPI endpoint without making it an action. + +The `.EndpointDescriptor` wraps a function and marks it to be added to the +HTTP API at the same time as the properties and actions of the host `.Thing`. +This is intended to allow flexibility to implement endpoints that cannot be +described in a Thing Description as actions or properties. + +It may use any `fastapi` responses or arguments, as it passes keyword +arguments through to the relevant `fastapi` decorator. + +This will most usually be applied as a decorator with arguments, available +as :deco:`.fastapi_endpoint`. See the documentation for that function for +more detail. +""" + from __future__ import annotations from functools import partial, wraps @@ -19,10 +34,11 @@ from ..thing import Thing HTTPMethod = Literal["get", "post", "put", "delete"] +"""Valid HTTP verbs to use with `.fastapi_endpoint` or `.EndpointDescriptor`.""" class EndpointDescriptor: - """A descriptor to allow Things to easily add other endpoints""" + """A descriptor to allow Things to easily add other endpoints.""" def __init__( self, @@ -31,22 +47,49 @@ def __init__( path: Optional[str] = None, **kwargs: Mapping, ): + r"""Initialise an EndpointDescriptor. + + See `.fastapi_endpoint`, which is the usual way of instantiating this + class. + + :param func: is the method (defined on a `.Thing`) wrapped by this + descriptor. + :param http_method: the HTTP verb we are responding to. This selects + the FastAPI decorator: ``"get"`` corresponds to ``@app.get``. + :param path: the URL, relative to the host `.Thing`, for the endpoint. + :param \**kwargs: additional keyword arguments are passed to the + FastAPI decorator, allowing you to specify responses, OpenAPI + parameters, etc. + """ self.func = func self.http_method = http_method self._path = path self.kwargs = kwargs @overload - def __get__(self, obj: Literal[None], type=None) -> Self: ... + def __get__(self, obj: Literal[None], type=None) -> Self: ... # noqa: D105 @overload - def __get__(self, obj: Thing, type=None) -> Callable: ... + def __get__(self, obj: Thing, type=None) -> Callable: ... # noqa: D105 + + def __get__( + self, obj: Optional[Thing], type: type[Thing] | None = None + ) -> Union[Self, Callable]: + """Bind the method to the host `.Thing` and return it. - def __get__(self, obj: Optional[Thing], type=None) -> Union[Self, Callable]: - """The function, bound to an object as for a normal method. + When called on a `.Thing`, this descriptor returns the wrapped + function, with the `.Thing` bound as its first argument. This is + the usual behaviour for Python methods. If `obj` is None, the descriptor is returned, so we can get the descriptor conveniently as an attribute of the class. + + :param obj: The `.Thing` on which the descriptor is defined, or ``None``. + :param type: The class on which the descriptor is defined. + + :return: The wrapped function, bound to the `.Thing` (when called as + an instance attribute), or the descriptor itself (when called as + a class attribute). """ if obj is None: return self @@ -54,26 +97,35 @@ def __get__(self, obj: Optional[Thing], type=None) -> Union[Self, Callable]: @property def name(self): - """The name of the wrapped function""" + """The name of the wrapped function.""" return self.func.__name__ @property def path(self): - """The path of the endpoint (relative to the Thing)""" + """The path of the endpoint (relative to the Thing).""" return self._path or self.name @property def title(self): - """A human-readable title""" + """A human-readable title.""" return get_summary(self.func) or self.name @property def description(self): - """A description of the endpoint""" + """A description of the endpoint.""" return get_docstring(self.func, remove_summary=True) def add_to_fastapi(self, app: FastAPI, thing: Thing): - """Add this function to a FastAPI app, bound to a particular Thing.""" + """Add an endpoint for this function to a FastAPI app. + + We will add an endpoint to the app, bound to a particular `.Thing`. + The URL will be prefixed with the `.Thing` path, i.e. the specified + URL (which defaults to the name of this descriptor) is relative to + the host `.Thing`. + + :param app: the `fastapi.FastAPI` application we are adding to. + :param thing: the `.Thing` we're bound to. + """ # fastapi_endpoint is equivalent to app.get/app.post/whatever fastapi_endpoint = getattr(app, self.http_method) bound_function = partial(self.func, thing) diff --git a/src/labthings_fastapi/descriptors/property.py b/src/labthings_fastapi/descriptors/property.py index f51a31cf..c8638a04 100644 --- a/src/labthings_fastapi/descriptors/property.py +++ b/src/labthings_fastapi/descriptors/property.py @@ -1,5 +1,11 @@ -""" -Define an object to represent an Action, as a descriptor. +"""Define a descriptor to represent properties. + +:ref:`wot_properties` are represented in LabThings by `.ThingProperty` descriptors. +These descriptors work similarly to regular Python properties or attributes, +with the addition of features that allow them to be accessed over HTTP and +documented in the :ref:`wot_td` and OpenAPI documents. + +This module defines the `.ThingProperty` class. """ from __future__ import annotations @@ -12,7 +18,7 @@ from ..utilities import labthings_data, wrap_plain_types_in_rootmodel from ..utilities.introspection import get_summary, get_docstring -from ..thing_description.model import PropertyAffordance, Form, DataSchema, PropertyOp +from ..thing_description._model import PropertyAffordance, Form, DataSchema, PropertyOp from ..thing_description import type_to_dataschema from ..exceptions import NotConnectedToServerError @@ -22,10 +28,10 @@ class ThingProperty: - """A property that can be accessed via the HTTP API + """A property that can be accessed via the HTTP API. - By default, a ThingProperty is "dumb", i.e. it acts just like - a normal variable. + By default, a ThingProperty acts like + a normal variable, but functionality can be added in several ways. """ model: type[BaseModel] @@ -42,6 +48,62 @@ def __init__( getter: Optional[Callable] = None, setter: Optional[Callable] = None, ): + """Create a property that can be accessed via the HTTP API. + + `.ThingProperty` is a descriptor that functions like a variable, optionally + with notifications when it is set. It may also have a getter and setter, + which work in a similar way to Python properties. + + `.ThingProperty` can behave in several different ways: + + * If no ``getter`` or ``setter`` is specified, it will behave like a simple + data attribute (i.e. a variable). If ``observable`` is ``True``, it is + possible to register for notifications when the value is set. In this + case, an ``initial_value`` is required. + * If a ``getter`` is specified and ``observable`` is ``False``, the ``getter`` + will be called when the property is accessed, and its return value + will be the property's value, just like the builtin ``property``. The + property will be read-only both locally and via HTTP. + * If a ``getter`` is specified and ``observable`` is ``True``, the ``getter`` + is used instead of ``initial_value`` but thereafter the property + behaves like a variable. The ``getter`` is only on first access. + The property may be written to locally, and whether it's writable + via HTTP depends on the ``readonly`` argument. + * If both a ``getter`` and ``setter`` are specified and ``observable`` is + ``False``, + the property behaves like a Python property, with the ``getter`` being + called when the property is accessed, and the ``setter`` being called + when the property is set. The property is read-only via HTTP if + ``readonly`` is ``True``. It may always be written to locally. + * If ``observable`` is ``True`` and a ``setter`` is specified, the property + will behave like a variable, but will call the ``setter`` + when the property is set. The ``setter`` may perform tasks like sending + the updated value to the hardware, but it is not responsible for + remembering the value. The initial value is set via the ``getter`` or + ``initial_value``. + + + :param model: The type of the property. This is optional, because it is + better to use type hints (see notes on typing above). + :param initial_value: The initial value of the property. If this is set, + the property must not have a getter, and should behave like a variable. + :param readonly: If True, the property cannot be set via the HTTP API. + :param observable: If True, the property can be observed for changes via + websockets. This causes the setter to run code in the async event loop + that will notify a list of subscribers each time the property is set. + Currently, only websockets can be used to observe properties. + :param description: A description of the property, used in the API + documentation. LabThings will attempt to take this from the docstring + if not supplied. + :param title: A human-readable title for the property, used in the API + documentation. Defaults to the first line of the docstring, or the name + of the property. + :param getter: A function that gets the value of the property. + :param setter: A function that sets the value of the property. + + :raise ValueError: if the initial value or type are missing or incorrectly + specified. + """ if getter and initial_value is not None: raise ValueError("getter and an initial value are mutually exclusive.") if model is None: @@ -59,12 +121,19 @@ def __init__( # link to the offending ThingProperty type_to_dataschema(self.model) - def __set_name__(self, owner, name: str): + def __set_name__(self, owner: type[Thing], name: str) -> None: + """Take note of the name to which the descriptor is assigned. + + This is called when the descriptor is assigned to an attribute of a class. + + :param owner: the `.Thing` subclass to which we are being attached. + :param name: the name to which we have been assigned. + """ self._name = name @property def title(self): - """A human-readable title""" + """A human-readable title for the property.""" if self._title: return self._title if self._getter and get_summary(self._getter): @@ -73,11 +142,11 @@ def title(self): @property def description(self): - """A description of the property""" + """A description of the property.""" return self._description or get_docstring(self._getter, remove_summary=True) - def __get__(self, obj, type=None) -> Any: - """The value of the property + def __get__(self, obj: Thing | None, type: type | None = None) -> Any: + """Return the value of the property. If `obj` is none (i.e. we are getting the attribute of the class), we return the descriptor. @@ -87,6 +156,12 @@ def __get__(self, obj, type=None) -> Any: If a getter is set, we will use it, unless the property is observable, at which point the getter is only ever used once, to set the initial value. + + :param obj: the `.Thing` to which we are attached. + :param type: the class on which we are defined. + + :return: the value of the property (when accessed on an instance), or + this descriptor if accessed as a class attribute. """ if obj is None: return self @@ -104,31 +179,46 @@ def __get__(self, obj, type=None) -> Any: else: return self.initial_value - def __set__(self, obj, value): - """Set the property's value""" + def __set__(self, obj: Thing, value: Any) -> None: + """Set the property's value. + + :param obj: the `.Thing` to which we are attached. + :param value: the new value for the property. + """ obj.__dict__[self.name] = value if self._setter: self._setter(obj, value) self.emit_changed_event(obj, value) - def _observers_set(self, obj): - """A set used to notify changes""" + def _observers_set(self, obj: Thing): + """Return the observers of this property. + + Each observer in this set will be notified when the property is changed. + See ``.ThingProperty.emit_changed_event`` + + :param obj: the `.Thing` to which we are attached. + + :return: the set of observers corresponding to ``obj``. + """ ld = labthings_data(obj) if self.name not in ld.property_observers: ld.property_observers[self.name] = WeakSet() return ld.property_observers[self.name] def emit_changed_event(self, obj: Thing, value: Any) -> None: - """Notify subscribers that the property has changed + """Notify subscribers that the property has changed. This function is run when properties are upadated. It must be run from within a thread. This could be the `Invocation` thread of a running action, or the property should be updated over via a client/http. It must be run from a thread as it is communicating with the event loop via an `asyncio` blocking - portal. + portal and can cause deadlock if run in the event loop. + + :param obj: the `.Thing` to which we are attached. + :param value: the new property value, to be sent to observers. - :raises NotConnectedToServerError: if the Thing that is calling the property - update is not connected to a server with a running event loop. + :raise NotConnectedToServerError: if the Thing that is calling the property + update is not connected to a server with a running event loop. """ runner = obj._labthings_blocking_portal if not runner: @@ -145,7 +235,14 @@ def emit_changed_event(self, obj: Thing, value: Any) -> None: ) async def emit_changed_event_async(self, obj: Thing, value: Any): - """Notify subscribers that the property has changed""" + """Notify subscribers that the property has changed. + + This function may only be run in the `anyio` event loop. See + `.ThingProperty.emit_changed_event`. + + :param obj: the `.Thing` to which we are attached. + :param value: the new property value, to be sent to observers. + """ for observer in self._observers_set(obj): await observer.send( {"messageType": "propertyStatus", "data": {self._name: value}} @@ -153,11 +250,19 @@ async def emit_changed_event_async(self, obj: Thing, value: Any): @property def name(self): - """The name of the property""" + """The name of the property. + + This should be consistent between the class definition and the + :ref:`wot_td` as well as appearing in the URLs for getting and setting. + """ return self._name - def add_to_fastapi(self, app: FastAPI, thing: Thing): - """Add this action to a FastAPI app, bound to a particular Thing.""" + def add_to_fastapi(self, app: FastAPI, thing: Thing) -> None: + """Add this action to a FastAPI app, bound to a particular Thing. + + :param app: The FastAPI application we are adding endpoints to. + :param thing: The `.Thing` we are adding the endpoints for. + """ # We can't use the decorator in the usual way, because we'd need to # annotate the type of `body` with `self.model` which is only defined # at runtime. @@ -192,7 +297,14 @@ def get_property(): def property_affordance( self, thing: Thing, path: Optional[str] = None ) -> PropertyAffordance: - """Represent the property in a Thing Description.""" + """Represent the property in a Thing Description. + + :param thing: the `.Thing` to which we are attached. + :param path: the URL of the `.Thing`. If not present, we will retrieve + the ``path`` from ``thing``. + + :return: A description of the property in :ref:`wot_td` format. + """ path = path or thing.path ops = [PropertyOp.readproperty] if not self.readonly: @@ -222,16 +334,25 @@ def property_affordance( ) def getter(self, func: Callable) -> Self: - """set the function that gets the property's value""" + """Set the function that gets the property's value. + + :param func: is the new getter function. + + :return: this property (to allow its use as a decorator). + """ self._getter = func return self def setter(self, func: Callable) -> Self: - """Decorator to set the property's value + """Change the setter function. - ``ThingProperty`` descriptors return the value they hold + `.ThingProperty` descriptors return the value they hold when they are accessed. However, they can run code when they are set: this decorator sets a function as that code. + + :param func: is the new setter function. + + :return: this property (to allow its use as a decorator). """ self._setter = func self.readonly = False @@ -239,10 +360,12 @@ def setter(self, func: Callable) -> Self: class ThingSetting(ThingProperty): - """A setting can be accessed via the HTTP API and is persistent between sessions + """A `.ThingProperty` that persists on disk. + + A setting can be accessed via the HTTP API and is persistent between sessions. - A ThingSetting is a ThingProperty with extra functionality for triggering - a Thing to save its settings. + A `.ThingSetting` is a `.ThingProperty` with extra functionality for triggering + a `.Thing` to save its settings. Note: If a setting is mutated rather than assigned to, this will not trigger saving. For example: if a Thing has a setting called `dictsetting` holding the dictionary @@ -253,17 +376,26 @@ class ThingSetting(ThingProperty): The setting otherwise acts just like a normal variable. """ - def __set__(self, obj, value): - """Set the property's value""" + def __set__(self, obj: Thing, value: Any): + """Set the setting's value. + + This will cause the settings to be saved to disk. + + :param obj: the `.Thing` to which we are attached. + :param value: the new value of the setting. + """ super().__set__(obj, value) obj.save_settings() - def set_without_emit(self, obj, value): - """Set the property's value, but do not emit event to notify the server + def set_without_emit(self, obj: Thing, value: Any): + """Set the property's value, but do not emit event to notify the server. This function is not expected to be used externally. It is called during initial setup so that the setting can be set from disk before the server is fully started. + + :param obj: the `.Thing` to which we are attached. + :param value: the new value of the setting. """ obj.__dict__[self.name] = value if self._setter: diff --git a/src/labthings_fastapi/example_things/__init__.py b/src/labthings_fastapi/example_things/__init__.py index 61d4d7ff..4d4a1bdd 100644 --- a/src/labthings_fastapi/example_things/__init__.py +++ b/src/labthings_fastapi/example_things/__init__.py @@ -1,9 +1,11 @@ -""" -Example Thing subclasses, used for testing and demonstration purposes. +"""Example Thing subclasses, used for testing and demonstration purposes. + +Most of these are broken in some way and used for testing. These should be +moved into the unit tests. """ import time -from typing import Optional, Annotated +from typing import Any, Optional, Annotated from labthings_fastapi.thing import Thing from labthings_fastapi.decorators import thing_action, thing_property from labthings_fastapi.descriptors import ThingProperty @@ -11,7 +13,7 @@ class MyThing(Thing): - """An example Thing with a few affordances""" + """An example Thing with a few affordances.""" @thing_action def anaction( @@ -30,7 +32,7 @@ def anaction( ), ] = None, ) -> dict[str, str]: - """Quite a complicated action + """Quite a complicated action. This action has lots of parameters and is designed to confuse my schema generator. I hope it doesn't! @@ -38,7 +40,17 @@ def anaction( I might even use some Markdown here: * If this renders, it supports lists - * With at east two items. + * With at least two items. + + There is also a parameter and return block to satisfy docstring validators. + This may be preferable to annotations on the arguments. + + :param repeats: How many times to do it. + :param undocumented: There's no description on this field's type hint. + :param title: A human-readable title. + :param attempts: A list of names of attempts. + + :return: A dictionary with strings as keys and values. """ # We should be able to call actions as normal Python functions self.increment_counter() @@ -50,7 +62,12 @@ def make_a_dict( extra_key: Optional[str] = None, extra_value: Optional[str] = None, ) -> dict[str, Optional[str]]: - """An action that returns a dict""" + """Do something that returns a dict. + + :param extra_key: An additional key. + :param extra_value: An additional value. + :return: a dictionary. + """ out: dict[str, Optional[str]] = {"key": "value"} if extra_key is not None: out[extra_key] = extra_value @@ -58,7 +75,7 @@ def make_a_dict( @thing_action def increment_counter(self): - """Increment the counter property + """Increment the counter property. This action doesn't do very much - all it does, in fact, is increment the counter (which may be read using the @@ -68,7 +85,11 @@ def increment_counter(self): @thing_action def slowly_increase_counter(self, increments: int = 60, delay: float = 1): - """Increment the counter slowly over a minute""" + """Increment the counter slowly over a minute. + + :param increments: how many times to increment. + :param delay: the wait time between increments. + """ for i in range(increments): time.sleep(delay) self.increment_counter() @@ -85,41 +106,64 @@ def slowly_increase_counter(self, increments: int = 60, delay: float = 1): @thing_action def action_without_arguments(self) -> None: - """An action that takes no arguments""" + """Do something that takes no arguments.""" pass @thing_action - def action_with_only_kwargs(self, **kwargs) -> None: - """An action that takes **kwargs""" + def action_with_only_kwargs(self, **kwargs: dict) -> None: + r"""Do something that takes \**kwargs. + + :param \**kwargs: Keyword arguments. + """ pass class ThingWithBrokenAffordances(Thing): - """A Thing that raises exceptions in actions/properites""" + """A Thing that raises exceptions in actions/properites.""" @thing_action def broken_action(self): - """An action that raises an exception""" + """Do something that raises an exception. + + :raise RuntimeError: every time. + """ raise RuntimeError("This is a broken action") @thing_property def broken_property(self): - """A property that raises an exception""" + """Raise an exception when the property is accessed. + + :raise RuntimeError: every time. + """ raise RuntimeError("This is a broken property") class ThingThatCantInstantiate(Thing): - """A Thing that raises an exception in __init__""" + """A Thing that raises an exception in __init__.""" def __init__(self): + """Fail to initialise. + + :raise RuntimeError: every time. + """ raise RuntimeError("This thing can't be instantiated") class ThingThatCantStart(Thing): - """A Thing that raises an exception in __enter__""" + """A Thing that raises an exception in __enter__.""" def __enter__(self): + """Fail to start the thing. + + :raise RuntimeError: every time. + """ raise RuntimeError("This thing can't start") - def __exit__(self, exc_t, exc_v, exc_tb): + def __exit__(self, exc_t: Any, exc_v: Any, exc_tb: Any): + """Don't leave the thing as we never entered. + + :param exc_t: Exception type. + :param exc_v: Exception value. + :param exc_tb: Traceback. + """ pass diff --git a/src/labthings_fastapi/exceptions.py b/src/labthings_fastapi/exceptions.py index dfe93ce5..3fa595ae 100644 --- a/src/labthings_fastapi/exceptions.py +++ b/src/labthings_fastapi/exceptions.py @@ -1,10 +1,10 @@ -"""A submodule for custom LabThings-FastAPI Exceptions""" +"""A submodule for custom LabThings-FastAPI Exceptions.""" from .dependencies.invocation import InvocationCancelledError class NotConnectedToServerError(RuntimeError): - """The Thing is not connected to a server + """The Thing is not connected to a server. This exception is called if a ThingAction is called or is a ThingProperty is updated on a Thing that is not diff --git a/src/labthings_fastapi/notifications.py b/src/labthings_fastapi/notifications.py index 0730d745..b6a3052e 100644 --- a/src/labthings_fastapi/notifications.py +++ b/src/labthings_fastapi/notifications.py @@ -1,5 +1,4 @@ -""" -Handle notification of events, property, and action status changes +"""Handle notification of events, property, and action status changes. There are several kinds of "event" in the WoT vocabulary, not all of which are called Event, which is why this module is called `notifications`. @@ -20,4 +19,6 @@ class Listener: + """A placeholder class for objects that listen for notifications.""" + pass diff --git a/src/labthings_fastapi/outputs/__init__.py b/src/labthings_fastapi/outputs/__init__.py index e022ed1c..bf94e676 100644 --- a/src/labthings_fastapi/outputs/__init__.py +++ b/src/labthings_fastapi/outputs/__init__.py @@ -1,3 +1,9 @@ +"""Support for additional output formats. + +Currently, this submodule provides an MJPEG Stream output. See +`.MJPEGStreamDescriptor`. +""" + from .mjpeg_stream import MJPEGStream, MJPEGStreamDescriptor # __all__ enables convenience imports from this module. diff --git a/src/labthings_fastapi/outputs/blob.py b/src/labthings_fastapi/outputs/blob.py index 23ff1f8f..08236b66 100644 --- a/src/labthings_fastapi/outputs/blob.py +++ b/src/labthings_fastapi/outputs/blob.py @@ -1,45 +1,41 @@ -""" -# BLOB Output Module +"""BLOB Output Module. -The BlobOutput class is used when you need to return something file-like that can't +The ``.Blob`` class is used when you need to return something file-like that can't easily (or efficiently) be converted to JSON. This is useful for returning large objects like images, especially where an existing file-type is the obvious way to handle it. -There is a [dedicated documentation page on blobs](/blobs.rst) that explains how to use +There is a documentation page on :ref:`blobs` that explains how to use this mechanism. -To return a file from an action, you should declare its return type as a BlobOutput +To return a file from an action, you should declare its return type as a `.Blob` subclass, defining the -[`media_type`](#labthings_fastapi.outputs.blob.Blob.media_type) attribute. +`.Blob.media_type` attribute. + +.. code-block:: python -```python -class MyImageBlob(Blob): - media_type = "image/png" + class MyImageBlob(Blob): + media_type = "image/png" -class MyThing(Thing): - @thing_action - def get_image(self) -> MyImageBlob: - # Do something to get the image data - data = self._get_image_data() - return MyImageBlob.from_bytes(data) -``` + + class MyThing(Thing): + @thing_action + def get_image(self) -> MyImageBlob: + # Do something to get the image data + data = self._get_image_data() + return MyImageBlob.from_bytes(data) The action should then return an instance of that subclass, with data supplied either as a `bytes` object or a file on disk. If files are used, it's your responsibility to ensure the file is deleted after the -[`Blob`](#labthings_fastapi.outputs.blob.Blob) object is -garbage-collected. Constructing it using the class methods -[`from_bytes`](#labthings_fastapi.outputs.blob.Blob.from_bytes) or -[`from_temporary_directory`](#labthings_fastapi.outputs.blob.Blob.from_temporary_directory) -will ensure this is done for you. - -Bear in mind a `tempfile` object only holds a file descriptor and is not safe for -concurrent use, which does not work well with the HTTP API: +`.Blob` object is garbage-collected. Constructing it using the class methods +`.Blob.from_bytes` or `.Blob.from_temporary_directory` will ensure this is +done for you. + +Bear in mind a `tempfile.TemporaryFile` object only holds a file descriptor +and is not safe for concurrent use, which does not work well with the HTTP API: action outputs may be retrieved multiple times after the action has completed, possibly concurrently. Creating a temp folder and making a file inside it -with -[`from_temporary_directory`](#labthings_fastapi.outputs.blob.Blob.from_temporary_directory) -is the safest way to deal with this. +with `.Blob.from_temporary_directory` is the safest way to deal with this. """ from __future__ import annotations @@ -50,6 +46,7 @@ def get_image(self) -> MyImageBlob: import shutil from typing import ( Annotated, + AsyncGenerator, Callable, Literal, Mapping, @@ -77,12 +74,12 @@ def get_image(self) -> MyImageBlob: class BlobData(Protocol): """The interface for the data store of a Blob. - [`Blob`](#labthings_fastapi.outputs.blob.Blob) objects can represent their data in various ways. Each of + `.Blob` objects can represent their data in various ways. Each of those options must provide three ways to access the data, which are the `content` property, the `save()` method, and the `open()` method. This protocol defines the interface needed by any data store used by a - [`Blob`](#labthings_fastapi.outputs.blob.Blob). + `.Blob`. Objects that are used on the server will additionally need to implement the [`ServerSideBlobData`](#labthings_fastapi.outputs.blob.ServerSideBlobData) protocol, @@ -91,81 +88,148 @@ class BlobData(Protocol): @property def media_type(self) -> str: - """The MIME type of the data, e.g. 'image/png' or 'application/json'""" + """The MIME type of the data, e.g. 'image/png' or 'application/json'.""" pass @property def content(self) -> bytes: - """The data as a `bytes` object""" + """The data as a `bytes` object.""" pass def save(self, filename: str) -> None: - """Save the data to a file""" + """Save the data to a file. + + :param filename: the path where the file should be saved. + """ ... def open(self) -> io.IOBase: - """Return a file-like object that may be read from.""" + """Return a file-like object that may be read from. + + :return: an open file-like object. + """ ... class ServerSideBlobData(BlobData, Protocol): - """A BlobData protocol for server-side use, i.e. including `response()` + """A BlobData protocol for server-side use, i.e. including `response()`. - [`Blob`](#labthings_fastapi.outputs.blob.Blob) objects returned by actions must use - [`BlobData`](#labthings_fastapi.outputs.blob.BlobData) objects - that can be downloaded. This protocol extends that protocol to - include a [`response()`](#labthings_fastapi.outputs.blob.ServerSideBlobData.response) method that returns a FastAPI response object. + `.Blob` objects returned by actions must use `.BlobData` objects + that can be downloaded. This protocol extends the `.BlobData` protocol to + include a `.ServerSideBlobData.response` method that returns a + `fastapi.Response` object. - See [`BlobBytes`](#labthings_fastapi.outputs.blob.BlobBytes) or - [`BlobFile`](#labthings_fastapi.outputs.blob.BlobFile) for concrete implementations. + See `.BlobBytes` or `.BlobFile` for concrete implementations. """ id: Optional[uuid.UUID] = None """A unique identifier for this BlobData object. - + The ID is set when the BlobData object is added to the BlobDataManager. It is used to retrieve the BlobData object from the manager. """ def response(self) -> Response: - """A :class:`fastapi.Response` object that sends binary data.""" + """Return a`fastapi.Response` object that sends binary data. + + :return: a response that streams the data from disk or memory. + """ ... class BlobBytes: - """A BlobOutput that holds its data in memory as a :class:`bytes` object""" + """A `.Blob` that holds its data in memory as a `bytes` object. + + `.Blob` objects use objects conforming to the `.BlobData` protocol to + store their data either on disk or in a file. This implements the protocol + using a `bytes` object in memory. + + .. note:: + + This class is rarely instantiated directly. It is usually best to use + `.Blob.from_bytes` on a `.Blob` subclass. + """ id: Optional[uuid.UUID] = None + """A unique ID to identify the data in a `.BlobManager`.""" def __init__(self, data: bytes, media_type: str): + """Create a `.BlobBytes` object. + + `.BlobBytes` objects wrap data stored in memory as `bytes`. They + are not usually instantiated directly, but made using `.Blob.from_bytes`. + + :param data: is the data to be wrapped. + :param media_type: is the MIME type of the data. + """ self._bytes = data self.media_type = media_type @property def content(self) -> bytes: + """The wrapped data, as a `bytes` object.""" return self._bytes def save(self, filename: str) -> None: + """Save the wrapped data to a file. + + :param filename: where to save the data. + """ with open(filename, "wb") as f: f.write(self._bytes) def open(self) -> io.IOBase: + """Return an open file-like object containing the data. + + This wraps the underlying `bytes` in an `io.BytesIO`. + + :return: an `io.BytesIO` object wrapping the data. + """ return io.BytesIO(self._bytes) def response(self) -> Response: + """Send the underlying data over the network. + + :return: a response that streams the data from memory. + """ return Response(content=self._bytes, media_type=self.media_type) class BlobFile: - """A BlobOutput that holds its data in a file + """A `.Blob` that holds its data in a file. + + `.Blob` objects use objects conforming to the `.BlobData` protocol to + store their data either on disk or in a file. This implements the protocol + using a file on disk. Only the filepath is retained by default. If you are using e.g. a temporary - directory, you should add the temporary directory as a property, to stop it - being garbage collected.""" + directory, you should add the `.TemporaryDirectory` as an instance attribute, + to stop it being garbage collected. See `.Blob.from_temporary_directory`. + + .. note:: + + This class is rarely instantiated directly. It is usually best to use + `.Blob.from_temporary_directory` on a `.Blob` subclass. + """ id: Optional[uuid.UUID] = None + """A unique ID to identify the data in a `.BlobManager`.""" def __init__(self, file_path: str, media_type: str, **kwargs): + r"""Create a `.BlobFile` to wrap data stored on disk. + + `.BlobFile` objects wrap data stored on disk as files. They + are not usually instantiated directly, but made using + `.Blob.from_temporary_directory` or `.Blob.from_file`. + + :param file_path: is the path to the file. + :param media_type: is the MIME type of the data. + :param \**kwargs: will be added to the object as instance + attributes. This may be used to stop temporary directories + from being garbage collected while the `.Blob` exists. + + :raise IOError: if the file specified does not exist. + """ if not os.path.exists(file_path): raise IOError("Tried to return a file that doesn't exist.") self._file_path = file_path @@ -175,69 +239,122 @@ def __init__(self, file_path: str, media_type: str, **kwargs): @property def content(self) -> bytes: + """The wrapped data, as a `bytes` object in memory. + + This reads the file on disk into a `bytes` object. + + :return: the contents of the file in a `bytes` object. + """ with open(self._file_path, "rb") as f: return f.read() def save(self, filename: str) -> None: + """Save the wrapped data to a file. + + `.BlobFile` objects already store their data on disk. + Currently, this method copies the file to the given + filename. In the future, this may change to ``move`` + for increased efficiency. + + :param filename: the path where the file should be saved. + """ shutil.copyfile(self._file_path, filename) def open(self) -> io.IOBase: + """Return an open file-like object containing the data. + + In the case of `.BlobFile`, this is an open file handle + to the underlying file, which is where the data is already + stored. It is opened with mode ``"rb"`` i.e. read-only and + binary. + + :return: an open file handle. + """ return open(self._file_path, mode="rb") def response(self) -> Response: + """Generate a response allowing the file to be downloaded. + + :return: a response that streams the file from disk. + """ return FileResponse(self._file_path, media_type=self.media_type) class Blob(BaseModel): - """A container for binary data that may be retrieved over HTTP + """A container for binary data that may be retrieved over HTTP. - See the [documentation on blobs](/blobs.rst) for more information on how to use this class. + See :ref:`blobs` for more information on how to use this class. - A [`Blob`](#labthings_fastapi.outputs.blob.Blob) may be created - to hold data using the class methods - `from_bytes` or `from_temporary_directory`. The constructor will - attempt to deserialise a Blob from a URL (see `__init__` method). + A `.Blob` may be created to hold data using the class methods + `.Blob.from_bytes`, `.Blob.from_file` or `.Blob.from_temporary_directory`. + The constructor will attempt to deserialise a Blob from a URL + (see `__init__` method) and is unlikely to be used except in code + internal to LabThings. - You are strongly advised to subclass this class and specify the - `media_type` attribute, as this will propagate to the auto-generated + You are strongly advised to use a subclass of this class that specifies the + `.Blob.media_type` attribute, as this will propagate to the auto-generated documentation. """ href: str - """The URL where the data may be retrieved. This will be `blob://local` - if the data is stored locally.""" + """The URL where the data may be retrieved. + + `.Blob` objects on a `.ThingServer` are assigned a URL when they are + serialised to JSON. This allows them to be downloaded as binary data in a + separate HTTP request. + + `.Blob` objects created by a `.ThingClient` contain a URL pointing to the + data, which will be downloaded when it is requred. + + `.Blob` objects that store their data in a file or in memory will have the + ``href`` attribute set to the special value `blob://local`. + """ media_type: str = "*/*" """The MIME type of the data. This should be overridden in subclasses.""" rel: Literal["output"] = "output" + """The relation of this link to the host object. + + Currently, `.Blob` objects are found in the output of :ref:`actions`, so they + always have ``rel = "output"``. + """ description: str = ( "The output from this action is not serialised to JSON, so it must be " "retrieved as a file. This link will return the file." ) + """This description is added to the serialised `.Blob`.""" _data: Optional[ServerSideBlobData] = None """This object holds the data, either in memory or as a file. - + If `_data` is `None`, then the Blob has not been deserialised yet, and the `href` should point to a valid address where the data may be downloaded. """ @model_validator(mode="after") - def retrieve_data(self): - """Retrieve the data from the URL + def retrieve_data(self) -> Self: + r"""Retrieve the data from the URL. - When a [`Blob`](#labthings_fastapi.outputs.blob.Blob) is created - using its constructor, [`pydantic`](https://docs.pydantic.dev/latest/) + When a `.Blob` is created using its constructor, `pydantic` will attempt to deserialise it by retrieving the data from the URL - specified in `href`. Currently, this must be a URL pointing to a - [`Blob`](#labthings_fastapi.outputs.blob.Blob) that already exists on - this server. + specified in `.Blob.href`. Currently, this must be a URL pointing to a + `.Blob` that already exists on this server, and any other URL will + cause a `LookupError`. This validator will only work if the function to resolve URLs to - [`BlobData`](#labthings_fastapi.outputs.blob.BlobData) objects - has been set in the context variable - [`url_to_blobdata_ctx`](#labthings_fastapi.outputs.blob.url_to_blobdata_ctx). + `.BlobData` objects + has been set in the context variable `.blob.url_to_blobdata_ctx`\ . This is done when actions are being invoked over HTTP by the - [`BlobIOContextDep`](#labthings_fastapi.outputs.blob.BlobIOContextDep) dependency. + `.BlobIOContextDep` dependency. + + :return: the `.Blob` object (i.e. ``self``), after retrieving the data. + + :raise ValueError: if the ``href`` is set as ``"blob://local"`` but + the ``_data`` attribute has not been set. This happens when the + `.Blob` is being constructed using `.Blob.from_bytes` or similar. + :raise LookupError: if the `.Blob` is being constructed from a URL + and the URL does not correspond to a `.BlobData` instance that + exists on this server (i.e. one that has been previously created + and added to the `.BlobManager` as the result of a previous action). """ if self.href == "blob://local": if self._data: @@ -256,20 +373,25 @@ def retrieve_data(self): @model_serializer(mode="plain", when_used="always") def to_dict(self) -> Mapping[str, str]: - """Serialise the Blob to a dictionary and make it downloadable + r"""Serialise the Blob to a dictionary and make it downloadable. - When [`pydantic`](https://docs.pydantic.dev/latest/) serialises this object, + When `pydantic` serialises this object, it will call this method to convert it to a dictionary. There is a significant side-effect, which is that we will add the blob to the - [`BlobDataManager`](#labthings_fastapi.outputs.blob.BlobDataManager) so it - can be downloaded. + `.BlobDataManager` so it can be downloaded. This serialiser will only work if the function to assign URLs to - [`BlobData`](#labthings_fastapi.outputs.blob.BlobData) objects - has been set in the context variable - [`blobdata_to_url_ctx`](#labthings_fastapi.outputs.blob.blobdata_to_url_ctx). + `.BlobData` objects has been set in the context variable + `.blobdata_to_url_ctx`\ . This is done when actions are being returned over HTTP by the - [`BlobIOContextDep`](#labthings_fastapi.outputs.blob.BlobIOContextDep) dependency. + `.BlobIOContextDep` dependency. + + :return: a JSON-serialisable dictionary with a URL that allows + the `.Blob` to be downloaded from the `.BlobManager`. + + :raise LookupError: if the context variable providing access to the + `.BlobManager` is not available. This usually means the `.Blob` is + being serialised somewhere other than the output of an action. """ if self.href == "blob://local": try: @@ -292,29 +414,35 @@ def to_dict(self) -> Mapping[str, str]: @classmethod def default_media_type(cls) -> str: - """The default media type. + """Return the default media type. - `Blob` should generally be subclassed to define the default media type, + `.Blob` should generally be subclassed to define the default media type, as this forms part of the auto-generated documentation. Using the - `Blob` class directly will result in a media type of `*/*`, which makes + `.Blob` class directly will result in a media type of `*/*`, which makes it unclear what format the output is in. + + :return: the default media type as a MIME type string, e.g. ``image/png``. """ return cls.model_fields["media_type"].get_default() @property def data(self) -> ServerSideBlobData: - """The data store for this Blob + """The data store for this Blob. + + `.Blob` objects may hold their data in various ways, defined by the + `.ServerSideBlobData` protocol. This property returns the data store + for this `.Blob`. + + If the `.Blob` has not yet been downloaded, there may be no data + held locally, in which case this function will raise an exception. - `Blob` objects may hold their data in various ways, defined by the - [`ServerSideBlobData`](#labthings_fastapi.outputs.blob.ServerSideBlobData) - protocol. This property returns the data store for this `Blob`. + It is recommended to use the `.Blob.content` property or `.Blob.save` + or `.Blob.open` + methods rather than accessing this property directly. - If the `Blob` has not yet been downloaded, there may be no data - held locally, in which case this function will raise a `ValueError`. + :return: the data store wrapping data on disk or in memory. - It is recommended to use the `content` property or `save()` or `open()` - methods rather than accessing this property directly. Those methods will - download data if required, rather than raising an error. + :raise ValueError: if there is no data stored on disk or in memory. """ if self._data is None: raise ValueError("This Blob has no data.") @@ -322,7 +450,7 @@ def data(self) -> ServerSideBlobData: @property def content(self) -> bytes: - """Return the the output as a `bytes` object + """Return the the output as a `bytes` object. This property may return the `bytes` object, or if we have a file it will read the file and return the contents. Client objects may use @@ -331,23 +459,44 @@ def content(self) -> bytes: This property is read-only. You should also only read it once, as no guarantees are given about cacheing - reading it many times risks reading the file from disk many times, or re-downloading an artifact. + + :return: a `bytes` object containing the data. """ return self.data.content def save(self, filepath: str) -> None: """Save the output to a file. - This may remove the need to hold the output in memory. + This may remove the need to hold the output in memory, especially + if it is already stored on disk. + + :param filepath: The location to save the data on disk. """ self.data.save(filepath) def open(self) -> io.IOBase: - """Open the output as a binary file-like object.""" + """Open the data as a binary file-like object. + + This will return a file-like object that may be read from. It may be + either on disk (i.e. an open file handle) or in memory (e.g. an + `io.BytesIO` wrapper). + + :return: a binary file-like object. + """ return self.data.open() @classmethod def from_bytes(cls, data: bytes) -> Self: - """Create a BlobOutput from a bytes object""" + """Create a `.Blob` from a bytes object. + + This is the recommended way to create a `.Blob` from data that is held + in memory. It should ideally be called on a subclass that has set the + ``media_type``. + + :param data: the data as a `bytes` object. + + :return: a `.Blob` wrapping the supplied data. + """ return cls.model_construct( # type: ignore[return-value] href="blob://local", _data=BlobBytes(data, media_type=cls.default_media_type()), @@ -355,11 +504,22 @@ def from_bytes(cls, data: bytes) -> Self: @classmethod def from_temporary_directory(cls, folder: TemporaryDirectory, file: str) -> Self: - """Create a BlobOutput from a file in a temporary directory + """Create a `.Blob` from a file in a temporary directory. + + This is the recommended way to create a `.Blob` from data that is + saved to a file, when the file should not be retained. + It should ideally be called on a subclass that has set the + ``media_type``. - The TemporaryDirectory object will persist as long as this BlobOutput does, - which will prevent it from being cleaned up until the object is garbage - collected. + The `tempfile.TemporaryDirectory` object will persist as long as this + `.Blob` does, which will prevent it from being cleaned up until the object + is garbage collected. This means the file will stay on disk until it is + no longer needed. + + :param folder: a `tempfile.TemporaryDirectory` where the file is saved. + :param file: the path to the file, relative to the ``folder``. + + :return: a `.Blob` wrapping the file. """ file_path = os.path.join(folder.name, file) return cls.model_construct( # type: ignore[return-value] @@ -374,33 +534,56 @@ def from_temporary_directory(cls, folder: TemporaryDirectory, file: str) -> Self @classmethod def from_file(cls, file: str) -> Self: - """Create a BlobOutput from a regular file + """Create a `.Blob` from a regular file. + + This is the recommended way to create a `.Blob` from a file, if that + file will persist on disk. It should ideally be called on a subclass + of `.Blob` that has set ``media_type``. + + .. note:: - The file should exist for at least as long as the BlobOutput does; this - is assumed to be the case and nothing is done to ensure it's not - temporary. If you are using temporary files, consider creating your - Blob with `from_temporary_directory` instead. + The file should exist for at least as long as the `.Blob` does; this + is assumed to be the case and nothing is done to ensure it's not + temporary. If you are using temporary files, consider creating your + `.Blob` with `from_temporary_directory` instead. + + :param file: is the path to the file. This file must exist. + + :return: a `.Blob` object referencing the specified file. """ return cls.model_construct( # type: ignore[return-value] href="blob://local", _data=BlobFile(file, media_type=cls.default_media_type()), ) - def response(self): - """ "Return a suitable response for serving the output""" + def response(self) -> Response: + """Return a suitable response for serving the output. + + This method is called by the `.ThingServer` to generate a response + that returns the data over HTTP. + + :return: an HTTP response that streams data from memory or file. + """ return self.data.response() def blob_type(media_type: str) -> type[Blob]: - """Create a BlobOutput subclass for a given media type + r"""Create a `.Blob` subclass for a given media type. This convenience function may confuse static type checkers, so it is usually clearer to make a subclass instead, e.g.: - ```python - class MyImageBlob(Blob): - media_type = "image/png" - ``` + .. code-block:: python + + class MyImageBlob(Blob): + media_type = "image/png" + + :param media_type: will be the default value of the ``media_type`` property + on the `.Blob` subclass. + + :return: a subclass of `.Blob` with the specified default media type. + + :raise ValueError: if the media type contains ``'`` or ``\``. """ if "'" in media_type or "\\" in media_type: raise ValueError("media_type must not contain single quotes or backslashes") @@ -412,22 +595,47 @@ class MyImageBlob(Blob): class BlobDataManager: - """A class to manage BlobData objects + r"""A class to manage BlobData objects. - The `BlobManager` is responsible for serving `Blob` objects to clients. It - holds weak references: it will not retain `Blob`s that are no longer in use. - Most `Blob`s will be retained by the output of an action: this holds a strong - reference, and will be expired by the - [`ActionManager`](#labthings_fastapi.actions.ActionManager). - """ + The `.BlobManager` is responsible for serving `.Blob` objects to clients. It + holds weak references: it will not retain `.Blob`\ s that are no longer in use. + Most `.Blob`\ s will be retained by the output of an action: this holds a strong + reference, and will be expired by the `.ActionManager`. - _blobs: WeakValueDictionary[uuid.UUID, ServerSideBlobData] + Note that the `.BlobDataManager` does not work with `.Blob` objects directly, + it holds only the `.ServerSideBlobData` object, which is where the data is + stored. This means you should not rely on any custom attributes of a `.Blob` + subclass being preserved when the `.Blob` is passed from one action to another. - def __init__(self): - self._blobs = WeakValueDictionary() + See :ref:`blobs` for an overview of how `.Blob` objects should be used. + """ + + def __init__(self) -> None: + """Initialise a BlobDataManager object.""" + self._blobs: WeakValueDictionary[uuid.UUID, ServerSideBlobData] = ( + WeakValueDictionary() + ) def add_blob(self, blob: ServerSideBlobData) -> uuid.UUID: - """Add a BlobOutput to the manager, generating a unique ID""" + """Add a `.Blob` to the manager, generating a unique ID. + + This function adds a `.ServerSideBlobData` object to the + `.BlobDataManager`. It will retain a weak reference to the + `.ServerSideBlobData` object: you are responsible for ensuring + the data is not garbage collected, for example by including the + parent `.Blob` in the output of an action. + + :param blob: a `.ServerSideBlobData` object that holds the data + being added. + + :return: a unique ID identifying the data. This forms part of + the URL to download the data. + + :raise ValueError: if the `.ServerSideBlobData` object already + has an ``id`` attribute but is not in the dictionary of + data. This suggests the object has been added to another + `.BlobDataManager`, which should never happen. + """ if hasattr(blob, "id") and blob.id is not None: if blob.id in self._blobs: return blob.id @@ -441,43 +649,93 @@ def add_blob(self, blob: ServerSideBlobData) -> uuid.UUID: return blob.id def get_blob(self, blob_id: uuid.UUID) -> ServerSideBlobData: - """Retrieve a BlobOutput from the manager""" + """Retrieve a `.Blob` from the manager. + + :param blob_id: the unique ID assigned when the data was added to + this `.BlobDataManager`. + + :return: the `.ServerSideBlobData` object holding the data. + """ return self._blobs[blob_id] - def download_blob(self, blob_id: uuid.UUID): - """Download a BlobOutput""" + def download_blob(self, blob_id: uuid.UUID) -> Response: + """Download a `.Blob`. + + This function returns a `fastapi.Response` allowing the data to be + downloaded, using the `.ServerSideBlobData.response` method. + + :param blob_id: the unique ID assigned when the data was added to + this `.BlobDataManager`. + + :return: a `fastapi.Response` object that will send the content of + the blob over HTTP. + """ blob = self.get_blob(blob_id) return blob.response() - def attach_to_app(self, app: FastAPI): - """Attach the BlobDataManager to a FastAPI app""" + def attach_to_app(self, app: FastAPI) -> None: + """Attach the BlobDataManager to a FastAPI app. + + Add an endpoint to a FastAPI application that will serve the content of + the `.ServerSideBlobData` objects in response to ``GET`` requests. + + :param app: the `fastapi.FastAPI` application to which we are adding + the endpoint. + """ app.get("/blob/{blob_id}")(self.download_blob) blobdata_to_url_ctx = ContextVar[Callable[[ServerSideBlobData], str]]("blobdata_to_url") """This context variable gives access to a function that makes BlobData objects -downloadable, by assigning a URL and adding them to the +downloadable, by assigning a URL and adding them to the [`BlobDataManager`](#labthings_fastapi.outputs.blob.BlobDataManager). -It is only available within a +It is only available within a [`blob_serialisation_context_manager`](#labthings_fastapi.outputs.blob.blob_serialisation_context_manager) because it requires access to the `BlobDataManager` and the `url_for` function from the FastAPI app. """ -url_to_blobdata_ctx = ContextVar[Callable[[str], BlobData]]("url_to_blobdata") +url_to_blobdata_ctx = ContextVar[Callable[[str], ServerSideBlobData]]("url_to_blobdata") """This context variable gives access to a function that makes BlobData objects from a URL, by retrieving them from the [`BlobDataManager`](#labthings_fastapi.outputs.blob.BlobDataManager). -It is only available within a +It is only available within a [`blob_serialisation_context_manager`](#labthings_fastapi.outputs.blob.blob_serialisation_context_manager) because it requires access to the `BlobDataManager`. """ -async def blob_serialisation_context_manager(request: Request): - """Set context variables to allow blobs to be [de]serialised""" +async def blob_serialisation_context_manager( + request: Request, +) -> AsyncGenerator[BlobDataManager]: + r"""Set context variables to allow blobs to be [de]serialised. + + In order to serialise a `.Blob` to a JSON-serialisable dictionary, we must + add it to the `.BlobDataManager` and use that to generate a URL. This + requres that the serialisation code (which may be nested deep within a + `pydantic.BaseModel`) has access to the `.BlobDataManager` and also the + `fastapi.Request.url_for` method. At time of writing, there was not an + obvious way to pass these functions in to the serialisation code. + + Similar problems exist for blobs used as input: the validator needs to + retrieve the data from the `.BlobDataManager` but does not have access. + + This async context manager yields the `.BlobDataManager`, but more + importantly it sets the `.url_to_blobdata_ctx` and `blobdata_to_url_ctx` + context variables, which may be accessed by the code within `.Blob` to + correctly add and retrieve `.ServerSideBlobData` objects to and from the + `.BlobDataManager`\ . + + This function will usually be called from a FastAPI dependency. See + :ref:`dependencies` for more on that mechanism. + + :param request: the `fastapi.Request` object, used to access the server + and ``url_for`` method. + + :yield: the `.BlobDataManager`. This is usually ignored. + """ thing_server = find_thing_server(request.app) blob_manager: BlobDataManager = thing_server.blob_data_manager url_for = request.url_for @@ -486,7 +744,7 @@ def blobdata_to_url(blob: ServerSideBlobData) -> str: blob_id = blob_manager.add_blob(blob) return str(url_for("download_blob", blob_id=blob_id)) - def url_to_blobdata(url: str) -> BlobData: + def url_to_blobdata(url: str) -> ServerSideBlobData: m = re.search(r"blob/([0-9a-z\-]+)", url) if not m: raise HTTPException( @@ -507,4 +765,4 @@ def url_to_blobdata(url: str) -> BlobData: BlobIOContextDep: TypeAlias = Annotated[ BlobDataManager, Depends(blob_serialisation_context_manager) ] -"""A dependency that enables `Blob`s to be serialised and deserialised.""" +"""A dependency that enables `.Blob` to be serialised and deserialised.""" diff --git a/src/labthings_fastapi/outputs/mjpeg_stream.py b/src/labthings_fastapi/outputs/mjpeg_stream.py index 9ffd9aeb..c4cfb03f 100644 --- a/src/labthings_fastapi/outputs/mjpeg_stream.py +++ b/src/labthings_fastapi/outputs/mjpeg_stream.py @@ -1,9 +1,16 @@ +"""MJPEG Stream support. + +This module defines a descriptor that allows `.Thing` subclasses to expose an +MJPEG stream. See `.MJPEGStreamDescriptor`. +""" + from __future__ import annotations from dataclasses import dataclass from datetime import datetime from fastapi import FastAPI from fastapi.responses import StreamingResponse, HTMLResponse from typing import ( + Any, AsyncGenerator, AsyncIterator, Literal, @@ -26,18 +33,36 @@ @dataclass class RingbufferEntry: - """A single entry in a ringbuffer""" + """A single entry in a ringbuffer. + + This structure comprises one frame as a JPEG, plus a timestamp and + a buffer index. Each time a frame is added to the stream, it is + tagged with a timestamp and index, with the index increasing by + 1 each time. + """ frame: bytes + """The frame as a `bytes` object, which is a JPEG image for an MJPEG stream.""" timestamp: datetime + """The time the frame was added to the ringbuffer.""" index: int + """The index of the frame within the stream.""" class MJPEGStreamResponse(StreamingResponse): + """A StreamingResponse that streams an MJPEG stream. + + This response uses an async generator that yields `bytes` + objects, each of which is a JPEG file. We add the --frame markers and mime + types that mark it as an MJPEG stream. This is sufficient to enable it to + work in an `img` tag, with the `src` set to the MJPEG stream's endpoint. + """ + media_type = "multipart/x-mixed-replace; boundary=frame" + """The media_type used to describe the endpoint in FastAPI.""" def __init__(self, gen: AsyncGenerator[bytes, None], status_code: int = 200): - """A StreamingResponse that streams an MJPEG stream + """Set up StreamingResponse that streams an MJPEG stream. This response is initialised with an async generator that yields `bytes` objects, each of which is a JPEG file. We add the --frame markers and mime @@ -49,6 +74,11 @@ def __init__(self, gen: AsyncGenerator[bytes, None], status_code: int = 200): NB the ``status_code`` argument is used by FastAPI to set the status code of the response in OpenAPI. + + :param gen: an async generator, yielding `bytes` objects each of which is + one image, in JPEG format. + :param status_code: The status code associated with the response, by default + a 200 code is returned. """ self.frame_async_generator = gen StreamingResponse.__init__( @@ -59,7 +89,14 @@ def __init__(self, gen: AsyncGenerator[bytes, None], status_code: int = 200): ) async def mjpeg_async_generator(self) -> AsyncGenerator[bytes, None]: - """A generator yielding an MJPEG stream""" + """Return a generator yielding an MJPEG stream. + + This async generator wraps each incoming JPEG frame with the + ``--frame`` separator and content type header. It is the basis + of the response sent over HTTP (see ``__init__``). + + :yield: JPEG frames, each with a ``--frame`` marker prepended. + """ async for frame in self.frame_async_generator: yield b"--frame\r\nContent-Type: image/jpeg\r\n\r\n" yield frame @@ -67,7 +104,7 @@ async def mjpeg_async_generator(self) -> AsyncGenerator[bytes, None]: class MJPEGStream: - """Manage streaming images over HTTP as an MJPEG stream + """Manage streaming images over HTTP as an MJPEG stream. An MJPEGStream object handles accepting images (already in JPEG format) and streaming them to HTTP clients as a multipart @@ -77,15 +114,25 @@ class MJPEGStream: call `add_frame` with JPEG image data. To add a stream to a `.Thing`, use the `.MJPEGStreamDescriptor` - which will handle creating an `MJPEGStream` object on first access, + which will handle creating an `.MJPEGStream` object on first access, and will also add it to the HTTP API. The MJPEG stream buffers the last few frames (10 by default) and also has a hook to notify the size of each frame as it is added. - The latter is used by OpenFlexure's autofocus routine. + The latter is used by OpenFlexure's autofocus routine. The + ringbuffer is intended to support clients receiving notification + of new frames, and then retrieving the frame (shortly) afterwards. """ def __init__(self, ringbuffer_size: int = 10): + """Initialise an MJPEG stream. + + See the class docstring for `.MJPEGStream`. Note that it will + often be initialised by `.MJPEGStreamDescriptor`. + + :param ringbuffer_size: The number of frames to retain in + memory, to allow retrieval after the frame has been sent. + """ self._lock = threading.Lock() self.condition = anyio.Condition() self._streaming = False @@ -93,7 +140,12 @@ def __init__(self, ringbuffer_size: int = 10): self.reset(ringbuffer_size=ringbuffer_size) def reset(self, ringbuffer_size: Optional[int] = None): - """Reset the stream and optionally change the ringbuffer size""" + """Reset the stream and optionally change the ringbuffer size. + + Discard all frames from the ringbuffer and reset the frame index. + + :param ringbuffer_size: the number of frames to keep in memory. + """ with self._lock: self._streaming = True n = ringbuffer_size or len(self._ringbuffer) @@ -107,16 +159,31 @@ def reset(self, ringbuffer_size: Optional[int] = None): ] self.last_frame_i = -1 - def stop(self, portal: BlockingPortal): - """Stop the stream""" + def stop(self, portal: BlockingPortal) -> None: + """Stop the stream. + + Stop the stream and cause all clients to disconnect. + + :param portal: an `anyio.from_thread.BlockingPortal` that allows + this function to use the event loop to notify that the stream + should stop. + """ with self._lock: self._streaming = False portal.start_task_soon(self.notify_stream_stopped) async def ringbuffer_entry(self, i: int) -> RingbufferEntry: - """Return the ith frame acquired by the camera + """Return the ith frame acquired by the camera. - :param i: The index of the frame to read + The ringbuffer means we can retrieve frames even if they are not + the latest frame. Specifying ``i`` also makes it simple to ensure + that every frame in a stream is acquired. + + :param i: The index of the frame to read. + + :return: the frame, together with a timestamp and its index. + + :raise ValueError: if the frame is not available. """ if i < 0: raise ValueError("i must be >= 0") @@ -132,17 +199,39 @@ async def ringbuffer_entry(self, i: int) -> RingbufferEntry: @asynccontextmanager async def buffer_for_reading(self, i: int) -> AsyncIterator[bytes]: - """Yields the ith frame as a bytes object + """Yield the ith frame as a bytes object. + + Retrieve frame ``i`` from the ringbuffer. + + This allows async code access to a frame in the ringbuffer. + The frame will not be copied, and should not be written to. + The frame may not exist after the function has completed (i.e. + after any ``with`` statement has finished). + + Using a context manager is intended to allow future versions of this + code to manage access to the ringbuffer (e.g. allowing buffer re-use). + Currently, buffers are always created as fresh `bytes` objects, so + this context manager does not provide additional functionality + over `.MJPEGStream.ringbuffer_entry`. :param i: The index of the frame to read + + :yield: The frame's data as `bytes`, along with timestamp and index. """ entry = await self.ringbuffer_entry(i) yield entry.frame async def next_frame(self) -> int: - """Wait for the next frame, and return its index + """Wait for the next frame, and return its index. - :raises StopAsyncIteration: if the stream has stopped.""" + This async function will yield until a new frame arrives, then return + its index. The index may then be used to retrieve the new frame + with `.MJPEGStream.buffer_for_reading`. + + :return: the index of the next frame to arrive. + + :raise StopAsyncIteration: if the stream has stopped. + """ async with self.condition: await self.condition.wait() if not self._streaming: @@ -150,26 +239,46 @@ async def next_frame(self) -> int: return self.last_frame_i async def grab_frame(self) -> bytes: - """Wait for the next frame, and return it + """Wait for the next frame, and return it. - This copies the frame for safety, so we can release the - read lock on the buffer. + This copies the frame for safety, so there is no need to release + or return the buffer. + + :return: The next JPEG frame, as a `bytes` object. """ i = await self.next_frame() async with self.buffer_for_reading(i) as frame: return copy(frame) async def next_frame_size(self) -> int: - """Wait for the next frame and return its size + """Wait for the next frame and return its size. This is useful if you want to use JPEG size as a sharpness metric. + + :return: The size of the next JPEG frame, in bytes. """ i = await self.next_frame() async with self.buffer_for_reading(i) as frame: return len(frame) async def frame_async_generator(self) -> AsyncGenerator[bytes, None]: - """A generator that yields frames as bytes""" + """Yield frames as bytes objects. + + This generator will return frames from the MJPEG stream. These are + taken from the ringbuffer by `.MJPEGStream.buffer_for_reading` and + so should have any buffer-management considerations taken care of. + + Code using this generator should complete as quickly as possible, + because future implementations may hold a lock while this function + yields. If lengthy processing is required, please copy the buffer + and continue processing elsewhere. + + Note that this will wait for a new frame each time. There is no + guarantee that we won't skip frames. + + :yield: the frames in sequence, as a `bytes` object containing + JPEG data. + """ while self._streaming: try: i = await self.next_frame() @@ -182,20 +291,42 @@ async def frame_async_generator(self) -> AsyncGenerator[bytes, None]: return async def mjpeg_stream_response(self) -> MJPEGStreamResponse: - """Return a StreamingResponse that streams an MJPEG stream""" + """Return a StreamingResponse that streams an MJPEG stream. + + This wraps each frame with the required header to make the + multipart stream work, and sends it to the client via a + streaming response. It is sufficient to show up as a video + in an ``img`` tag, or to be streamed to disk as an MJPEG + format video. + + :return: a streaming response in MJPEG format. + """ return MJPEGStreamResponse(self.frame_async_generator()) def add_frame(self, frame: bytes, portal: BlockingPortal) -> None: - """Return the next buffer in the ringbuffer to write to + """Add a JPEG to the MJPEG stream. + + This function adds a frame to the stream. It may be called from + threaded code, but uses an `anyio.from_thread.BlockingPortal` to + call code in the `anyio` event loop, which is where notifications + are handled. :param frame: The frame to add :param portal: The blocking portal to use for scheduling tasks. This is necessary because tasks are handled asynchronously. The blocking portal may be obtained with a dependency, in `labthings_fastapi.dependencies.blocking_portal.BlockingPortal`. + + :raise ValueError: if the supplied frame does not start with the JPEG + start bytes and end with the end bytes. """ - assert frame[0] == 0xFF and frame[1] == 0xD8, ValueError("Invalid JPEG") - assert frame[-2] == 0xFF and frame[-1] == 0xD9, ValueError("Invalid JPEG") + if not ( + frame[0] == 0xFF + and frame[1] == 0xD8 + and frame[-2] == 0xFF + and frame[-1] == 0xD9 + ): + raise ValueError("Invalid JPEG") with self._lock: entry = self._ringbuffer[(self.last_frame_i + 1) % len(self._ringbuffer)] entry.timestamp = datetime.now() @@ -220,38 +351,55 @@ async def notify_stream_stopped(self) -> None: class MJPEGStreamDescriptor: - """A descriptor that returns a MJPEGStream object when accessed + """A descriptor that returns a MJPEGStream object when accessed. If this descriptor is added to a `.Thing`, it will create an `.MJPEGStream` object when it is first accessed. It will also add two HTTP endpoints, one with the name of the descriptor serving the MJPEG stream, and another with `/viewer` appended, which serves a basic HTML page that views the stream. + This descriptor does not currently show up in the :ref:`wot_td`. """ - def __init__(self, **kwargs): - self._kwargs = kwargs + def __init__(self, **kwargs: dict[str, Any]): + r"""Initialise an MJPEGStreamDescriptor. + + :param \**kwargs: keyword arguments are passed to the initialiser of + `.MJPEGStream`. + """ + self._kwargs: Any = kwargs + + def __set_name__(self, _owner: Thing, name: str) -> None: + """Remember the name to which we are assigned. + + The name is important, as it will set the URL of the HTTP endpoint used + to access the stream. - def __set_name__(self, owner, name): + :param _owner: the `.Thing` to which we are attached. + :param name: the name to which this descriptor is assigned. + """ self.name = name @overload - def __get__(self, obj: Literal[None], type=None) -> Self: ... + def __get__(self, obj: Literal[None], type=None) -> Self: ... # noqa: D105 @overload - def __get__(self, obj: Thing, type=None) -> MJPEGStream: ... + def __get__(self, obj: Thing, type=None) -> MJPEGStream: ... # noqa: D105 + + def __get__( + self, obj: Optional[Thing], type: type[Thing] | None = None + ) -> Union[MJPEGStream, Self]: + """Return the MJPEG Stream, or the descriptor object. - def __get__(self, obj: Optional[Thing], type=None) -> Union[MJPEGStream, Self]: - """The value of the property + When accessed on the class, this ``__get__`` method will return the descriptor + object. This allows LabThings to add it to the HTTP API. - If ``obj`` is none (i.e. we are getting the attribute of the class), - we return the descriptor. + When accessed on the object, an `.MJPEGStream` is returned. - If no getter is set, we'll return either the initial value, or the value - from the object's ``__dict__``, i.e. we behave like a variable. + :param obj: the host `.Thing`, or ``None`` if accessed on the class. + :param type: the class on which we are defined. - If a getter is set, we will use it, unless the property is observable, at - which point the getter is only ever used once, to set the initial value. + :return: an `.MJPEGStream`, or this descriptor. """ if obj is None: return self @@ -262,10 +410,39 @@ def __get__(self, obj: Optional[Thing], type=None) -> Union[MJPEGStream, Self]: return obj.__dict__[self.name] async def viewer_page(self, url: str) -> HTMLResponse: + """Generate a trivial viewer page for the stream. + + :param url: the URL of the stream. + + :return: a trivial HTML page that views the stream. + """ return HTMLResponse(f"") - def add_to_fastapi(self, app: FastAPI, thing: Thing): - """Add the stream to the FastAPI app""" + def add_to_fastapi(self, app: FastAPI, thing: Thing) -> None: + """Add the stream to the FastAPI app. + + We create two endpoints, one for the MJPEG stream (using the name of + the descriptor, relative to the host `.Thing`) and one serving a + basic viewer. + + The example code below would create endpoints at ``/camera/stream`` + and ``/camera/stream/viewer``. + + .. code-block:: python + + import labthings_fastapi as lt + + + class Camera(lt.Thing): + stream = MJPEGStreamDescriptor() + + + server = lt.ThingServer() + server.add_thing(Camera(), "/camera") + + :param app: the `fastapi.FastAPI` application to which we are being added. + :param thing: the host `.Thing` instance. + """ app.get( f"{thing.path}{self.name}", response_class=MJPEGStreamResponse, diff --git a/src/labthings_fastapi/server/__init__.py b/src/labthings_fastapi/server/__init__.py index 4d7a5290..1235d7fc 100644 --- a/src/labthings_fastapi/server/__init__.py +++ b/src/labthings_fastapi/server/__init__.py @@ -1,5 +1,13 @@ +"""Code supporting the LabThings server. + +LabThings wraps the `fastapi.FastAPI` application in a `.ThingServer`, which +provides the tools to serve and manage `.Thing` instances. + +See the :ref:`tutorial` for examples of how to set up a `.ThingServer`. +""" + from __future__ import annotations -from typing import Optional, Sequence, TypeVar +from typing import AsyncGenerator, Optional, Sequence, TypeVar import os.path import re @@ -15,8 +23,10 @@ ) from ..actions import ActionManager from ..thing import Thing -from ..thing_description.model import ThingDescription -from ..dependencies.thing_server import _thing_servers +from ..thing_description._model import ThingDescription +from ..dependencies.thing_server import _thing_servers # noqa: F401 + +# `_thing_servers` is used as a global from `ThingServer.__init__` from ..outputs.blob import BlobDataManager # A path should be made up of names separated by / as a path separator. @@ -26,7 +36,37 @@ class ThingServer: + """Use FastAPI to serve `.Thing` instances. + + The `.ThingServer` sets up a `fastapi.FastAPI` application and uses it + to expose the capabilities of `.Thing` instances over HTTP. + + There are several functions of a `.ThingServer`: + + * Manage where settings are stored, to allow `.Thing` instances to + load and save their settings from disk. + * Configure the server to allow cross-origin requests (required if + we use a web app that is not served from the `.ThingServer`). + * Manage the threads used to run :ref:`actions`. + * Manage :ref:`blobs` to allow binary data to be returned. + * Allow threaded code to call functions in the event loop, by providing + an `anyio.from_thread.BlockingPortal`. + """ + def __init__(self, settings_folder: Optional[str] = None): + """Initialise a LabThings server. + + Setting up the `.ThingServer` involves creating the underlying + `fastapi.FastAPI` app, setting its lifespan function (used to + set up and shut down the `.Thing` instances), and configuring it + to allow cross-origin requests. + + We also create the `.ActionManager` to manage :ref:`actions` and the + `.BlobManager` to manage the downloading of :ref:`blobs`. + + :param settings_folder: the location on disk where `.Thing` + settings will be saved. + """ self.app = FastAPI(lifespan=self.lifespan) self.set_cors_middleware() self.settings_folder = settings_folder or "./settings" @@ -38,7 +78,7 @@ def __init__(self, settings_folder: Optional[str] = None): self._things: dict[str, Thing] = {} self.blocking_portal: Optional[BlockingPortal] = None self.startup_status: dict[str, str | dict] = {"things": {}} - global _thing_servers + global _thing_servers # noqa: F824 _thing_servers.add(self) app: FastAPI @@ -46,6 +86,15 @@ def __init__(self, settings_folder: Optional[str] = None): blob_data_manager: BlobDataManager def set_cors_middleware(self) -> None: + """Configure the server to allow requests from other origins. + + This is required to allow web applications access to the HTTP API, + if they are not served from the same origin (i.e. if they are not + served as part of the `.ThingServer`.). + + This is usually needed during development, and may be needed at + other times depending on how you are using LabThings. + """ self.app.add_middleware( CORSMiddleware, allow_origins=["*"], @@ -56,19 +105,36 @@ def set_cors_middleware(self) -> None: @property def things(self) -> Mapping[str, Thing]: - """Return a dictionary of all the things""" + """Return a dictionary of all the things. + + :return: a dictionary mapping thing paths to `.Thing` instances. + """ return MappingProxyType(self._things) ThingInstance = TypeVar("ThingInstance", bound=Thing) def things_by_class(self, cls: type[ThingInstance]) -> Sequence[ThingInstance]: - """Return all Things attached to this server matching a class""" + """Return all Things attached to this server matching a class. + + Return all instances of ``cls`` attached to this server. + + :param cls: A `.Thing` subclass. + + :return: all instances of ``cls`` that have been added to this server. + """ return [t for t in self.things.values() if isinstance(t, cls)] def thing_by_class(self, cls: type[ThingInstance]) -> ThingInstance: - """The Thing attached to this server matching a given class. + """Return the instance of ``cls`` attached to this server. + + This function calls `.ThingServer.things_by_class`, but asserts that + there is exactly one match. + + :param cls: a `.Thing` subclass. + + :return: the instance of ``cls`` attached to this server. - A RuntimeError will be raised if there is not exactly one matching Thing. + :raise RuntimeError: if there is not exactly one matching Thing. """ instances = self.things_by_class(cls) if len(instances) == 1: @@ -77,12 +143,15 @@ def thing_by_class(self, cls: type[ThingInstance]) -> ThingInstance: f"There are {len(instances)} Things of class {cls}, expected 1." ) - def add_thing(self, thing: Thing, path: str): - """Add a thing to the server + def add_thing(self, thing: Thing, path: str) -> None: + """Add a thing to the server. - :param thing: The thing to add to the server. + :param thing: The `.Thing` instance to add to the server. :param path: the relative path to access the thing on the server. Must only - contain alphanumeric characters, hyphens, or underscores. + contain alphanumeric characters, hyphens, or underscores. + + :raise ValueError: if ``path`` contains invalid characters. + :raise KeyError: if a `.Thing` has already been added at ``path``. """ # Ensure leading and trailing / if not path.endswith("/"): @@ -105,23 +174,33 @@ def add_thing(self, thing: Thing, path: str): ) @asynccontextmanager - async def lifespan(self, app: FastAPI): - """Manage set up and tear down + async def lifespan(self, app: FastAPI) -> AsyncGenerator[None]: + """Manage set up and tear down of the server and Things. + + This method is used as a lifespan function for the FastAPI app. See + the lifespan_ page in FastAPI's documentation. + + .. _lifespan: https://fastapi.tiangolo.com/advanced/events/#lifespan-function This does two important things: * It sets up the blocking portal so background threads can run async code - (important for events) - * It runs setup/teardown code for Things. + (this is required for events, streams, etc.). + * It runs setup/teardown code for Things by calling them as context + managers. + :param app: The FastAPI application wrapped by the server. + :yield: no value. The FastAPI application will serve requests while this + function yields. """ async with BlockingPortal() as portal: self.blocking_portal = portal # We attach a blocking portal to each thing, so that threaded code can # make callbacks to async code (needed for events etc.) for thing in self.things.values(): - if thing._labthings_blocking_portal is not None: - raise RuntimeError("Things may only ever have one blocking portal") + assert thing._labthings_blocking_portal is None, ( + "Things may only ever have one blocking portal" + ) thing._labthings_blocking_portal = portal # we __aenter__ and __aexit__ each Thing, which will in turn call the # synchronous __enter__ and __exit__ methods if they exist, to initialise @@ -147,7 +226,20 @@ def add_things_view_to_app(self): response_model_by_alias=True, ) def thing_descriptions(request: Request) -> Mapping[str, ThingDescription]: - """A dictionary of all the things available from this server""" + """Describe all the things available from this server. + + This returns a dictionary, where the keys are the paths to each + `.Thing` attached to the server, and the values are :ref:`wot_td` documents + represented as `.ThingDescription` objects. These should enable + clients to see all the capabilities of the `.Thing` instances and + access them over HTTP. + + :param request: is supplied automatically by FastAPI. + + :return: a dictionary mapping Thing paths to :ref:`wot_td` objects, which + are `pydantic.BaseModel` subclasses that get serialised to + dictionaries. + """ return { path: thing.thing_description(path, base=str(request.base_url)) for path, thing in thing_server.things.items() @@ -155,7 +247,13 @@ def thing_descriptions(request: Request) -> Mapping[str, ThingDescription]: @self.app.get("/things/") def thing_paths(request: Request) -> Mapping[str, str]: - """URLs pointing to the Thing Descriptions of each Thing.""" + """URLs pointing to the Thing Descriptions of each Thing. + + :param request: is supplied automatically by FastAPI. + + :return: a list of paths pointing to `.Thing` instances. These + URLs will return the :ref:`wot_td` of one `.Thing` each. + """ # noqa: D403 (URLs is correct capitalisation) return { t: f"{str(request.base_url).rstrip('/')}{t}" for t in thing_server.things.keys() @@ -163,7 +261,20 @@ def thing_paths(request: Request) -> Mapping[str, str]: def server_from_config(config: dict) -> ThingServer: - """Create a ThingServer from a configuration dictionary""" + """Create a ThingServer from a configuration dictionary. + + This function creates a `.ThingServer` and adds a number of `.Thing` + instances from a configuration dictionary. + + :param config: A dictionary, in the format used by :ref:`config_files` + + :return: A `.ThingServer` with instances of the specified `.Thing` + subclasses attached. The server will not be started by this + function. + + :raise ImportError: if a Thing could not be loaded from the specified + object reference. + """ server = ThingServer(config.get("settings_folder", None)) for path, thing in config.get("things", {}).items(): if isinstance(thing, str): @@ -176,10 +287,7 @@ def server_from_config(config: dict) -> ThingServer: f"specified as the class for {path}. The error is " f"printed below:\n\n{e}" ) - try: - instance = cls(*thing.get("args", {}), **thing.get("kwargs", {})) - except Exception as e: - raise e + instance = cls(*thing.get("args", {}), **thing.get("kwargs", {})) assert isinstance(instance, Thing), f"{thing['class']} is not a Thing" server.add_thing(instance, path) return server diff --git a/src/labthings_fastapi/server/cli.py b/src/labthings_fastapi/server/cli.py index 21e3eab8..068490b9 100644 --- a/src/labthings_fastapi/server/cli.py +++ b/src/labthings_fastapi/server/cli.py @@ -1,3 +1,23 @@ +"""Command-line interface to the `.ThingServer`. + +This module provides a command-line interface that is provided as +`labthings-server`. It exposes various functions that may be useful to +projects based on LabThings, if they wish to expose their own CLI. + +.. note:: + + In principle, LabThings may be run as an ASGI application wrapped + by a more advanced HTTP server providing HTTPS or other features. + This generally requires configuration via environment variables + rather than command-line flags. + + Environment variables are not yet supported, but may supplement + or replace the command line interface in the future. + +For examples of how to run the server from the command line, see +the tutorial page :ref:`tutorial_running`. +""" + from argparse import ArgumentParser, Namespace from typing import Optional import json @@ -11,11 +31,14 @@ def get_default_parser(): - """Return the default CLI parser for LabThings + """Return the default CLI parser for LabThings. - This can be used instead of `parse_args` if more arguments are needed - """ + This can be used to add more arguments, for custom CLIs that make use of + LabThings. + :return: an `argparse.ArgumentParser` set up with the options for + ``labthings-server``. + """ parser = ArgumentParser() parser.add_argument("-c", "--config", type=str, help="Path to configuration file") parser.add_argument("-j", "--json", type=str, help="Configuration as JSON string") @@ -37,14 +60,38 @@ def get_default_parser(): def parse_args(argv: Optional[list[str]] = None) -> Namespace: - """Process command line arguments for the server""" + r"""Process command line arguments for the server. + + The arguments are defined in `.get_default_parser`\ . + + :param argv: command line arguments (defaults to arguments supplied + to the current command). + + :return: a namespace with the extracted options. + """ parser = get_default_parser() # Use parser to parse CLI arguments and return the namespace with attributes set. return parser.parse_args(argv) def config_from_args(args: Namespace) -> dict: - """Process arguments and return a config dictionary""" + """Load the configuration from a supplied file or JSON string. + + This function will first attempt to load a JSON file specified in the + command line argument. It will then look for JSON configuration supplied + as a string. + + If both a file and a string are specified, the JSON string will be used + to ``update`` the configuration loaded from file, i.e. it will overwrite + keys in the file. + + :param args: Parsed arguments from `.parse_args`. + + :return: a server configuration, as a dictionary. + + :raise FileNotFoundError: if the configuration file specified is missing. + :raise RuntimeError: if neither a config file nor a string is provided. + """ if args.config: try: with open(args.config) as f: @@ -62,8 +109,34 @@ def config_from_args(args: Namespace) -> dict: return config -def serve_from_cli(argv: Optional[list[str]] = None, dry_run=False): - """Start the server from the command line""" +def serve_from_cli( + argv: Optional[list[str]] = None, dry_run: bool = False +) -> ThingServer | None: + r"""Start the server from the command line. + + This function will parse command line arguments, load configuration, + set up a server, and start it. It calls `.parse_args`, + `.config_from_args` and `.server_from_config` to get a server, then + starts `uvicorn` to serve on the specified host and port. + + If the ``fallback`` argument is specified, errors that stop the + LabThings server from starting will be handled by starting a simple + HTTP server that shows an error page. This behaviour may be helpful + if ``labthings-server`` is being run on a headless server, where + an HTTP error page is more useful than no response. + + :param argv: command line arguments (defaults to arguments supplied + to the current command). + :param dry_run: may be set to ``True`` to terminate after the server + has been created. This tests set-up code and verifies all of the + Things specified can be correctly loaded and instantiated, but + does not start `uvicorn`\ . + + :return: the `.ThingServer` instance created, if ``dry_run`` is ``True``. + + :raise BaseException: if the server cannot start, and the ``fallback`` + option is not specified. + """ args = parse_args(argv) try: config, server = None, None @@ -85,3 +158,4 @@ def serve_from_cli(argv: Optional[list[str]] = None, dry_run=False): uvicorn.run(app, host=args.host, port=args.port) else: raise e + return None # This is required as we sometimes return the server diff --git a/src/labthings_fastapi/server/fallback.py b/src/labthings_fastapi/server/fallback.py index 1b436aab..c40e9793 100644 --- a/src/labthings_fastapi/server/fallback.py +++ b/src/labthings_fastapi/server/fallback.py @@ -1,12 +1,32 @@ +"""A fallback server for LabThings. + +If the ``fallback`` option is given when ``labthings-server`` is run, we will +still start an HTTP server even if we cannot run LabThings with the specified +configuration. This means that something will still be viewable at the +expected URL, which is helpful if LabThings is running as a service, or +on embedded hardware. +""" + import json from traceback import format_exception +from typing import Any from fastapi import FastAPI from fastapi.responses import HTMLResponse from starlette.responses import RedirectResponse class FallbackApp(FastAPI): - def __init__(self, *args, **kwargs): + """A basic FastAPI application to serve a LabThings error page.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + r"""Set up a simple error server. + + This app is used to display a single page, which explains why the + LabThings server cannot start. + + :param \*args: is passed to `fastapi.FastAPI.__init__`\ . + :param \**kwargs: is passed to `fastapi.FastAPI.__init__`\ . + """ super().__init__(*args, **kwargs) self.labthings_config = None self.labthings_server = None @@ -50,7 +70,11 @@ def __init__(self, *args, **kwargs): @app.get("/") -async def root(): +async def root() -> HTMLResponse: + """Display the LabThings error page. + + :return: a response that serves the error as an HTML page. + """ error_message = f"{app.labthings_error}" # use traceback.format_exception to get full traceback as list # this ends in newlines, but needs joining to be a single string @@ -76,5 +100,14 @@ async def root(): @app.get("/{path:path}") -async def redirect_to_root(path: str): +async def redirect_to_root(path: str) -> RedirectResponse: + """Redirect all paths on the server to the error page. + + If any URL other than the error page is requested, this server will + redirect it to the error page. + + :param path: The path requested. + + :return: a response redirecting to the error page. + """ return RedirectResponse(url="/") diff --git a/src/labthings_fastapi/thing.py b/src/labthings_fastapi/thing.py index e9cc7a02..c84abb4c 100644 --- a/src/labthings_fastapi/thing.py +++ b/src/labthings_fastapi/thing.py @@ -1,21 +1,20 @@ -""" -The `Thing` class enables most of the functionality of this library, -and is the way in to most of its features. In the future, we might -support a stub version of the class in a separate package, so -that instrument control libraries can be LabThings compatible -without a hard dependency on LabThings. But that is something we -will do in the future... +"""A class to represent hardware or software Things. + +The `.Thing` class enables most of the functionality of this library, +and is the way in to most of its features. See :ref:`wot_cc` and :ref:`labthings_cc` +for more. """ from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Optional +from typing_extensions import Self from collections.abc import Mapping import logging import os import json from json.decoder import JSONDecodeError from fastapi.encoders import jsonable_encoder -from fastapi import Request +from fastapi import Request, WebSocket from anyio.abc import ObjectSendStream from anyio.from_thread import BlockingPortal from anyio.to_thread import run_sync @@ -23,11 +22,11 @@ from pydantic import BaseModel from .descriptors import ThingProperty, ThingSetting, ActionDescriptor -from .thing_description.model import ThingDescription, NoSecurityScheme +from .thing_description._model import ThingDescription, NoSecurityScheme from .utilities import class_attributes from .thing_description import validation from .utilities.introspection import get_summary, get_docstring -from .websockets import websocket_endpoint, WebSocket +from .websockets import websocket_endpoint if TYPE_CHECKING: @@ -38,73 +37,92 @@ class Thing: - """Represents a Thing, as defined by the Web of Things standard. + r"""Represents a Thing, as defined by the Web of Things standard. This class should encapsulate the code that runs a piece of hardware, or provides a particular function - it will correspond to a path on the server, and a Thing Description document. - ## Subclassing Notes - - * `__init__`: You should accept any arguments you need to configure the Thing - in `__init__`. Don't initialise any hardware at this time, as your Thing may - be instantiated quite early, or even at import time. - * `__enter__(self)` and `__exit__(self, exc_t, exc_v, exc_tb)` are where you - should start and stop communications with the hardware. This is Python's standard - "context manager" protocol. The arguments of `__exit__` will be `None` unless - an exception has occurred. You should be safe to ignore them, and just include - code that will close down your hardware. It's equivalent to a `finally:` block. - * Properties and Actions are defined using decorators: the `@thing_action` decorator - declares a method to be an action, which will run when it's triggered, and the - `@thing_property` decorator (or `ThingProperty` descriptor) does the same for - a property. See the documentation on those functions for more detail. + Subclassing Notes + ----------------- + + * ``__init__``: You should accept any arguments you need to configure the Thing + in ``__init__``. Don't initialise any hardware at this time, as your Thing may + be instantiated quite early, or even at import time. + * ``__enter__(self)`` and ``__exit__(self, exc_t, exc_v, exc_tb)`` are where you + should start and stop communications with the hardware. This is Python's + "context manager" protocol. The arguments of ``__exit__`` will be ``None`` + except after errors. You should be safe to ignore them, and just include + code that will close down your hardware, which is equivalent to a + ``finally:`` block. + * Properties and Actions are defined using decorators: the :deco:`.thing_action` + decorator declares a method to be an action, which will run when it's triggered, + and the :deco:`.thing_property` decorator (or `.ThingProperty` descriptor) does + the same for a property. See the documentation on those functions for more + detail. * `title` will be used in various places as the human-readable name of your Thing, - so it makes sense to set this in a subclass. + so it makes sense to set this in a subclass. - There are various LabThings methods that you should avoid overriding unless you know - what you are doing: anything not mentioned above that's defined in `Thing` is - probably best left along. They may in time be collected together into a single + There are various LabThings methods that you should avoid overriding unless you + know what you are doing: anything not mentioned above that's defined in `Thing` is + probably best left alone. They may in time be collected together into a single object to avoid namespace clashes. """ title: str + """A human-readable description of the Thing""" _labthings_blocking_portal: Optional[BlockingPortal] = None + """See :ref:`concurrency` for why blocking portal is needed.""" path: Optional[str] + """The path at which the `.Thing` is exposed over HTTP.""" - async def __aenter__(self): + async def __aenter__(self) -> Self: """Context management is used to set up/close the thing. As things (currently) do everything with threaded code, we define - async __aenter__ and __aexit__ wrappers to call the synchronous + async ``__aenter__`` and ``__aexit__`` wrappers to call the synchronous code, if it exists. + + :return: this object. """ if hasattr(self, "__enter__"): return await run_sync(self.__enter__) else: return self - async def __aexit__(self, exc_t, exc_v, exc_tb): + async def __aexit__( + self, exc_t: Any | None, exc_v: Any | None, exc_tb: Any + ) -> None: """Wrap context management functions, if they exist. - See __aenter__ docs for more details. + See ``__aenter__`` for more details. + + :param exc_t: The type of the exception, or ``None``. + :param exc_v: The exception that occurred, or ``None``. + :param exc_tb: The traceback for the exception, or ``None``. """ if hasattr(self, "__exit__"): - return await run_sync(self.__exit__, exc_t, exc_v, exc_tb) + await run_sync(self.__exit__, exc_t, exc_v, exc_tb) def attach_to_server( self, server: ThingServer, path: str, setting_storage_path: str - ): - """Attatch this thing to the server. + ) -> None: + """Attach this thing to the server. Things need to be attached to a server before use to function correctly. - :param server: The server to attach this Thing to - :param settings_storage_path: The path on disk to save the any Thing Settings - to. This should be the path to a json file. If it does not exist it will be - created. + :param server: The server to attach this Thing to. + :param path: The root URL for the Thing. + :param setting_storage_path: The path on disk to save the any Thing Settings + to. This should be the path to a json file. If it does not exist it will be + created. - Wc3 Web Of Things explanation: - This will add HTTP handlers to an app for all Interaction Affordances + Attaching the `.Thing` to a `.ThingServer` allows the `.Thing` to start + actions, load its settings from the correct place, and create HTTP endpoints + to allow it to be accessed from the HTTP API. + + We create HTTP endpoints for all :ref:`wot_affordances` on the `.Thing`, as well + as any `.EndpointDescriptor` descriptors. """ self.path = path self.action_manager: ActionManager = server.action_manager @@ -138,8 +156,8 @@ async def websocket(ws: WebSocket): _settings_store: Optional[dict[str, ThingSetting]] = None @property - def _settings(self) -> Optional[dict[str, ThingSetting]]: - """A private property that returns a dict of all settings for this Thing + def _settings(self) -> dict[str, ThingSetting]: + """A private property that returns a dict of all settings for this Thing. Each dict key is the name of the setting, the corresponding value is the ThingSetting class (a descriptor). This can be used to directly get the @@ -159,12 +177,33 @@ def _settings(self) -> Optional[dict[str, ThingSetting]]: @property def setting_storage_path(self) -> Optional[str]: - """The storage path for settings. This is set as the Thing is added to a server""" + """The storage path for settings. + + .. note:: + + This is set in `.Thing.attach_to_server`. It is ``None`` during the + ``__init__`` method, so it is best to avoid using settings until the + `.Thing` is set up in ``__enter__``. + """ return self._setting_storage_path - def load_settings(self, setting_storage_path): - """Load settings from json. This is run when the Thing is added to a server""" - # Ensure that the settings path isn't set during loading or saving will be triggered + def load_settings(self, setting_storage_path: str) -> None: + """Load settings from json. + + Read the JSON file and use it to populate settings. + + .. note:: + Settings are loaded when the Thing is added to a server, so they will + not be available while the ``__init__`` method is run. + + Note that no notifications will be triggered when the settings are set, + so if action is needed (e.g. updating hardware with the loaded settings) + it should be taken in ``__enter__``. + + :param setting_storage_path: The path where the settings should be stored. + """ + # Ensure that the settings path isn't set during loading or saving will be + # triggered self._setting_storage_path = None thing_name = type(self).__name__ if os.path.exists(setting_storage_path): @@ -176,7 +215,10 @@ def load_settings(self, setting_storage_path): self._settings[key].set_without_emit(self, value) else: _LOGGER.warning( - "Cannot set %s from persistent storage as %s has no matching setting.", + ( + "Cannot set %s from persistent storage as %s " + "has no matching setting." + ), key, thing_name, ) @@ -185,7 +227,11 @@ def load_settings(self, setting_storage_path): self._setting_storage_path = setting_storage_path def save_settings(self): - """Save settings to JSON. This is called whenever a setting is updated""" + """Save settings to JSON. + + This is called whenever a setting is updated. All settings are written to + the settings file every time. + """ if self._settings is not None: setting_dict = {} for name in self._settings.keys(): @@ -202,7 +248,7 @@ def save_settings(self): @property def thing_state(self) -> Mapping: - """Return a dictionary summarising our current state + """Return a dictionary summarising our current state. This is intended to be an easy way to collect metadata from a Thing that summarises its state. It might be used, for example, to record metadata @@ -219,8 +265,8 @@ def thing_state(self) -> Mapping: self._labthings_thing_state = {} return self._labthings_thing_state - def validate_thing_description(self): - """Raise an exception if the thing description is not valid""" + def validate_thing_description(self) -> None: + """Raise an exception if the thing description is not valid.""" td = self.thing_description_dict() return validation.validate_thing_description(td) @@ -231,12 +277,17 @@ def validate_thing_description(self): def thing_description( self, path: Optional[str] = None, base: Optional[str] = None ) -> ThingDescription: - """A w3c Thing Description representing this thing + """Generate a w3c Thing Description representing this thing. The w3c Web of Things working group defined a standard representation of a Thing, which provides a high-level description of the actions, properties, and events that it exposes. This endpoint delivers a JSON - representation of the Thing Description for this Thing. + representation of the :ref:`wot_td` for this Thing. + + :param path: the URL pointing to this Thing. + :param base: the base URL for all URLs in the thing description. + + :return: a Thing Description. """ path = path or getattr(self, "path", "{base_uri}") if ( @@ -270,26 +321,41 @@ def thing_description_dict( path: Optional[str] = None, base: Optional[str] = None, ) -> dict: - """A w3c Thing Description representing this thing, as a simple dict + r"""Describe this Thing with a Thing Description as a simple dict. - The w3c Web of Things working group defined a standard representation - of a Thing, which provides a high-level description of the actions, - properties, and events that it exposes. This endpoint delivers a JSON - representation of the Thing Description for this Thing. + See `.Thing.thing_description`\ . This function converts the + return value of that function into a simple dictionary. + + :param path: the URL pointing to this Thing. + :param base: the base URL for all URLs in the thing description. + + :return: a Thing Description. """ td: ThingDescription = self.thing_description(path=path, base=base) td_dict: dict = td.model_dump(exclude_none=True, by_alias=True) return jsonable_encoder(td_dict) - def observe_property(self, property_name: str, stream: ObjectSendStream): - """Register a stream to receive property change notifications""" + def observe_property(self, property_name: str, stream: ObjectSendStream) -> None: + """Register a stream to receive property change notifications. + + :param property_name: the property to register for. + :param stream: the stream used to send events. + + :raise KeyError: if the requested name is not defined on this Thing. + """ prop = getattr(self.__class__, property_name) if not isinstance(prop, ThingProperty): raise KeyError(f"{property_name} is not a LabThings Property") prop._observers_set(self).add(stream) def observe_action(self, action_name: str, stream: ObjectSendStream): - """Register a stream to receive action status change notifications""" + """Register a stream to receive action status change notifications. + + :param action_name: the action to register for. + :param stream: the stream used to send events. + + :raise KeyError: if the requested name is not defined on this Thing. + """ action = getattr(self.__class__, action_name) if not isinstance(action, ActionDescriptor): raise KeyError(f"{action_name} is not an LabThings Action") diff --git a/src/labthings_fastapi/thing_description/__init__.py b/src/labthings_fastapi/thing_description/__init__.py index 30db45ce..3774b64e 100644 --- a/src/labthings_fastapi/thing_description/__init__.py +++ b/src/labthings_fastapi/thing_description/__init__.py @@ -1,16 +1,16 @@ -""" -Thing Description module +"""Thing Description module. This module supports the generation of Thing Descriptions. Currently, the top -level function lives in `labthings_fastapi.thing.Thing.thing_description()`, +level function lives in `.Thing.thing_description`, but most of the supporting code is in this submodule. -A Pydantic model implementing the Thing Description is in `.model`, and this -is used to generate our TDs - it helps make sure any TD errors get caught when +A Pydantic model implementing the Thing Description is in +`.thing_description._model`, and this is used to generate our TDs - +using a `pydantic.BaseModel` helps make sure any TD errors get caught when they are generated in Python, which makes them much easier to debug. We also use the JSONSchema provided by W3C to validate the TDs we generate, in -`.validation`, as a double-check that we are standards-compliant. +`.thing_description.validation`, as a double-check that we are standards-compliant. """ from __future__ import annotations @@ -19,28 +19,49 @@ import json from pydantic import TypeAdapter, ValidationError -from .model import DataSchema +from ._model import DataSchema JSONSchema = dict[str, Any] # A type to represent JSONSchema def is_a_reference(d: JSONSchema) -> bool: - """Return True if a JSONSchema dict is a reference + """Return True if a JSONSchema dict is a reference. JSON Schema references are one-element dictionaries with a single key, `$ref`. `pydantic` sometimes breaks this - rule and so I don't check that it's a single key. + rule and so we don't check that it's a single key. + + :param d: A JSONSchema dictionary. + + :return: ``True`` if the dictionary contains ``$ref``. """ return "$ref" in d def look_up_reference(reference: str, d: JSONSchema) -> JSONSchema: - """Look up a reference in a JSONSchema + """Look up a reference in a JSONSchema. + + JSONSchema allows references, where chunks of JSON may be re-used. + Thing Description does not allow references, so we need to resolve + them and paste them in-line. + + This function can only deal with local references, i.e. they must + start with ``#`` indicating they belong to the current file. - This first asserts the reference is local (i.e. starts with # - so it's relative to the current file), then looks up - each path component in turn. + This function first asserts the reference is local + (i.e. starts with # so it's relative to the current file), + then looks up each path component in turn and returns the resolved + chunk of JSON. + + :param reference: the local reference (should start with ``#``). + :param d: the JSONSchema document. + + :return: the chunk of JSONSchema referenced by ``reference`` in ``d``. + + :raise KeyError: if the reference is not found in the supplied JSONSchema. + :raise NotImplementedError: if the reference does not start with ``"#/`` + and thus is not a local reference. """ if not reference.startswith("#/"): raise NotImplementedError( @@ -60,12 +81,27 @@ def look_up_reference(reference: str, d: JSONSchema) -> JSONSchema: def is_an_object(d: JSONSchema) -> bool: - """Determine whether a JSON schema dict is an object""" + """Determine whether a JSON schema dict is an object. + + :param d: a chunk of JSONSchema describing a datatype. + + :return: ``True`` if the ``type`` is ``object``. + """ return "type" in d and d["type"] == "object" def convert_object(d: JSONSchema) -> JSONSchema: - """Convert an object from JSONSchema to Thing Description""" + """Convert an object from JSONSchema to Thing Description. + + Convert JSONSchema objets to Thing Description datatypes. + + Currently, this deletes the ``additionalProperties`` keyword, which is + not supported by Thing Description. + + :param d: the JSONSchema object. + + :return: a copy of ``d``, with ``additionalProperties`` deleted. + """ out: JSONSchema = d.copy() # AdditionalProperties is not supported by Thing Description, and it is ambiguous # whether this implies it's false or absent. I will, for now, ignore it, so we @@ -76,12 +112,17 @@ def convert_object(d: JSONSchema) -> JSONSchema: def convert_anyof(d: JSONSchema) -> JSONSchema: - """Convert the anyof key to oneof + """Convert the anyof key to oneof. JSONSchema makes a distinction between "anyof" and "oneof", where the former means "any of these fields can be present" and the latter means "exactly one of these fields must be present". Thing Description does not have this - distinction, so we convert anyof to oneof. + distinction, so we convert ``anyof`` to ``oneof``. + + + :param d: the JSONSchema object. + + :return: a copy of ``d``, with ``anyOf`` replaced with ``oneOf``. """ if "anyOf" not in d: return d @@ -92,7 +133,7 @@ def convert_anyof(d: JSONSchema) -> JSONSchema: def convert_prefixitems(d: JSONSchema) -> JSONSchema: - """Convert the prefixitems key to items + """Convert the prefixitems key to items. JSONSchema 2019 (as used by thing description) used `items` with a list of values in the same way that JSONSchema @@ -104,19 +145,36 @@ def convert_prefixitems(d: JSONSchema) -> JSONSchema: additional items, and we raise a ValueError if that happens. This behaviour may be relaxed in the future. + + :param d: the JSONSchema object. + + :return: a copy of ``d``, converted to 2019 format as above. + + :raise KeyError: if we would overwrite an existing ``items`` + key. """ if "prefixItems" not in d: return d out: JSONSchema = d.copy() if "items" in out: - raise ValueError(f"Overwrote the `items` key on {out}.") + raise KeyError(f"Overwrote the `items` key on {out}.") out["items"] = out["prefixItems"] del out["prefixItems"] return out def convert_additionalproperties(d: JSONSchema) -> JSONSchema: - """Move additionalProperties into properties, or remove it""" + r"""Move additionalProperties into properties, or remove it. + + JSONSchema uses ``additionalProperties`` to define optional properties + of ``object``\ s. For Thing Descriptions, this should be moved inside + the ``properties`` object. + + :param d: the JSONSchema object. + + :return: a copy of ``d``, with ``additionalProperties`` moved into + ``properties`` or deleted if ``properties`` is not present. + """ if "additionalProperties" not in d: return d out: JSONSchema = d.copy() @@ -126,8 +184,14 @@ def convert_additionalproperties(d: JSONSchema) -> JSONSchema: return out -def check_recursion(depth: int, limit: int): - """Check the recursion count is less than the limit""" +def check_recursion(depth: int, limit: int) -> None: + """Check the recursion count is less than the limit. + + :param depth: the current recursion depth. + :param limit: the maximum recursion depth. + + :raise ValueError: if we exceed the recursion depth. + """ if depth > limit: raise ValueError( f"Recursion depth of {limit} exceeded - perhaps there is a circular " @@ -141,19 +205,42 @@ def jsonschema_to_dataschema( recursion_depth: int = 0, recursion_limit: int = 99, ) -> JSONSchema: - """remove references and change field formats + """Convert a data type description from JSONSchema to Thing Description. + + :ref:`wot_td` represents datatypes with DataSchemas, which are almost but not + quite JSONSchema format. There are two main tasks to convert them: + + Resolving references + -------------------- JSONSchema allows schemas to be replaced with `{"$ref": "#/path/to/schema"}`. Thing Description does not allow this. `dereference_jsonschema_dict` takes a `dict` representation of a JSON Schema document, and replaces all the references with the appropriate chunk of the file. + Converting union types + ---------------------- + JSONSchema can represent `Union` types using the `anyOf` keyword, which is called `oneOf` by Thing Description. It's possible to achieve the same thing in the specific case of array elements, by setting `items` to a list of `DataSchema` objects. This function does not yet do that conversion. This generates a copy of the document, to avoid messing up `pydantic`'s cache. + + This function runs recursively: to start with, only ``d`` should be provided + (the input JSONSchema). We will use the other arguments to keep track of + recursion as we convert the schema. + + :param d: a JSONSchema representation of a datatype. + :param root_schema: the whole JSONSchema document, for resolving references. + This will be set to ``d`` when the function is called initially. + :param recursion_depth: how deeply this function has recursed (starts at zero). + :param recursion_limit: how deeply this function is allowed to recurse. + + :return: the datatype in Thing Description format. This is not yet a + `.DataSchema` instance, but may be trivially converted to one + with ``DataSchema(**schema)``. """ root_schema = root_schema or d check_recursion(recursion_depth, recursion_limit) @@ -193,7 +280,7 @@ def jsonschema_to_dataschema( def type_to_dataschema(t: type, **kwargs) -> DataSchema: - """Convert a Python type to a Thing Description DataSchema + r"""Convert a Python type to a Thing Description DataSchema. This makes use of pydantic's `schema_of` function to create a json schema, then applies some fixes to make a DataSchema @@ -204,6 +291,15 @@ def type_to_dataschema(t: type, **kwargs) -> DataSchema: and will override the fields generated from the type that is passed in. Typically you'll want to use this for the `title` field. + + :param t: the Python datatype or `pydantic.BaseModel` subclass. + :param \**kwargs: Additional keyword arguments passed to the + `.DataSchema` constructor, often including ``title``. + + :return: a `.DataSchema` representing the type. + + :raise ValidationError: if the datatype cannot be represented + by a `.DataSchema`. """ if hasattr(t, "model_json_schema"): # The input should be a `BaseModel` subclass, in which case this works: diff --git a/src/labthings_fastapi/thing_description/model.py b/src/labthings_fastapi/thing_description/_model.py similarity index 73% rename from src/labthings_fastapi/thing_description/model.py rename to src/labthings_fastapi/thing_description/_model.py index 36eb295f..f5bdcf9a 100644 --- a/src/labthings_fastapi/thing_description/model.py +++ b/src/labthings_fastapi/thing_description/_model.py @@ -1,3 +1,19 @@ +"""Pydantic Models to describe DataSchema. + +:ref:`wot_td` defines a schema for describing Things, using JSONSchema-like syntax +for data types. This file contains pydantic models that describe that +schema. For the meaning of the various objects, please refer to the +td_schema_definition_ within the W3C standard. + +.. _td_schema_definition: https://www.w3.org/TR/wot-thing-description11/ + +This file was automatically generated, but has been customised to improve the +types and simplify/combine objects. + +I've added URLs to point to the schema documentation. These start with +``https://www.w3.org/TR/wot-thing-description11/``, meaning they refer to v1.1, +the version current when it was written. +""" # This file was generated by `datamodel-code-generator`, using # the command # datamodel-codegen --input person.json --input-file-type jsonschema --output model.py @@ -25,6 +41,11 @@ class Version(BaseModel): + """Version info for a Thing. + + See https://www.w3.org/TR/wot-thing-description11/#versioninfo. + """ + instance: str @@ -43,6 +64,8 @@ class Version(BaseModel): class Subprotocol(Enum): + """HTTP sub-protocols.""" + longpoll = "longpoll" websub = "websub" sse = "sse" @@ -59,6 +82,17 @@ class Subprotocol(Enum): def uses_thing_context(v: ThingContextType): + """Check the URLs in the ThingContextType are valid. + + This function makes ``assert`` statements, so will fail with an exception + if it is not valid. This module is hard coded to use valid URLs, so it is + not an expected error. + + See https://www.w3.org/TR/wot-thing-description11/#thing + This refers to the ``@context`` property. + + :param v: the ThingContextType object. + """ if not isinstance(v, list): assert v is THING_CONTEXT_URL else: @@ -76,6 +110,8 @@ def uses_thing_context(v: ThingContextType): class Type(Enum): + """``type`` property of a DataSchema.""" + boolean = "boolean" integer = "integer" number = "number" @@ -86,6 +122,11 @@ class Type(Enum): class DataSchema(BaseModel): + """Base for several classes describing datatypes. + + See https://www.w3.org/TR/wot-thing-description11/#dataschema. + """ + field_type: Optional[TypeDeclaration] = Field(None, alias="@type") description: Optional[Description] = None title: Optional[Title] = None @@ -125,91 +166,18 @@ class DataSchema(BaseModel): model_config = ConfigDict(extra="forbid") -""" -# The classes below attempted to implement the w3c spec for type-specific fields. -# However, this is very hard without complicated logic - and the w3c JSONSchema -# simply defines one DataSchema type, as I have done above. -# The code below almost but not quite works. - -class ArraySchema(DataSchema): - type: Type = Literal[Type.array] - items: Optional[Union[DataSchema, List[DataSchema]]] = None - maxItems: Optional[conint(ge=0)] = None - minItems: Optional[conint(ge=0)] = None - - -numberT = TypeVar("numberT", int, float) -class GenericNumberSchema(DataSchema, BaseModel, Generic[numberT]): - minimum: Optional[numberT] = None - maximum: Optional[numberT] = None - exclusiveMinimum: Optional[numberT] = None - exclusiveMaximum: Optional[numberT] = None - multipleOf: Optional[numberT] = None - - -class NumberSchema(GenericNumberSchema[float]): - type: Type = Literal[Type.number] - - -class IntegerSchema(GenericNumberSchema[int]): - type: Type = Literal[Type.integer] - - -class BooleanSchema(DataSchema): - type: Type = Literal[Type.boolean] - - -class ObjectSchema(DataSchema): - type: Type = Literal[Type.object] - properties: Optional[Mapping[str, DataSchema]] = None - required: Optional[List[str]] = None - - -class StringSchema(DataSchema): - type: Type = Literal[Type.string] - minLength: Optional[int] = None - maxLength: Optional[int] = None - pattern: Optional[str] = None - contentEncoding: Optional[str] = None - contentMediaType: Optional[str] = None - - -class NullSchema(DataSchema): - type: Type = Literal[Type.object] - - -DataSchema: Type = Union[ - DataSchema, - ArraySchema, - BooleanSchema, - IntegerSchema, - NullSchema, - NumberSchema, - ObjectSchema, - StringSchema, -] - - -DATA_SCHEMA_MODELS: list[BaseModel] = [ - DataSchema, - ArraySchema, - BooleanSchema, - IntegerSchema, - NullSchema, - NumberSchema, - ObjectSchema, - StringSchema, -] -for model in DATA_SCHEMA_MODELS: - model.update_forward_refs() -""" - - class Response(BaseModel): + """See https://www.w3.org/TR/wot-thing-description11/#expectedresponse.""" + contentType: Optional[str] = None class PropertyOp(Enum): + """``op`` values for Forms on Properties. + + See https://www.w3.org/TR/wot-thing-description11/#propertyaffordance + """ + readproperty = "readproperty" writeproperty = "writeproperty" observeproperty = "observeproperty" @@ -217,15 +185,30 @@ class PropertyOp(Enum): class ActionOp(Enum): + """``op`` values for Forms on Actions. + + See https://www.w3.org/TR/wot-thing-description11/#actionaffordance + """ + invokeaction = "invokeaction" class EventOp(Enum): + """``op`` values for Forms on Events. + + See https://www.w3.org/TR/wot-thing-description11/#eventaffordance + """ + subscribeevent = "subscribeevent" unsubscribeevent = "unsubscribeevent" class RootOp(Enum): + """``op`` values for Forms on Things. + + See https://www.w3.org/TR/wot-thing-description11/#thing + """ + readallproperties = "readallproperties" writeallproperties = "writeallproperties" readmultipleproperties = "readmultipleproperties" @@ -239,6 +222,8 @@ class RootOp(Enum): class Form(BaseModel, Generic[OpT]): + """See https://www.w3.org/TR/wot-thing-description11/#form.""" + model_config = ConfigDict(extra="allow") href: AnyUri @@ -252,6 +237,8 @@ class Form(BaseModel, Generic[OpT]): class InteractionAffordance(BaseModel): + """See https://www.w3.org/TR/wot-thing-description11/#interactionaffordance.""" + model_config = ConfigDict(extra="allow") description: Optional[Description] = None @@ -263,11 +250,15 @@ class InteractionAffordance(BaseModel): class PropertyAffordance(InteractionAffordance, DataSchema): + """See https://www.w3.org/TR/wot-thing-description11/#propertyaffordance.""" + observable: Optional[bool] = None forms: List[Form[PropertyOp]] = Field(..., min_length=1) class ActionAffordance(InteractionAffordance): + """See https://www.w3.org/TR/wot-thing-description11/#actionaffordance.""" + field_type: Optional[TypeDeclaration] = Field(None, alias="@type") input: Optional[DataSchema] = None output: Optional[DataSchema] = None @@ -277,6 +268,8 @@ class ActionAffordance(InteractionAffordance): class EventAffordance(BaseModel): + """See https://www.w3.org/TR/wot-thing-description11/#eventaffordance.""" + field_type: Optional[TypeDeclaration] = Field(None, alias="@type") subscription: Optional[DataSchema] = None data: Optional[DataSchema] = None @@ -285,6 +278,8 @@ class EventAffordance(BaseModel): class LinkElement(BaseModel): + """See https://www.w3.org/TR/wot-thing-description11/#link.""" + model_config = ConfigDict(extra="allow") href: AnyUri @@ -297,6 +292,8 @@ class LinkElement(BaseModel): class SecuritySchemeEnum(Enum): + """See https://www.w3.org/TR/wot-thing-description11/#securityscheme.""" + nosec = "nosec" # was Scheme basic = "basic" # was Scheme1 digest = "digest" # was Scheme2 @@ -307,6 +304,8 @@ class SecuritySchemeEnum(Enum): class In(Enum): + """Where a parameter is found.""" + header = "header" query = "query" body = "body" @@ -314,15 +313,21 @@ class In(Enum): class Qop(Enum): + """See https://www.w3.org/TR/wot-thing-description11/#digestsecurityscheme.""" + auth = "auth" auth_int = "auth-int" class Flow(Enum): + """See https://www.w3.org/TR/wot-thing-description11/#oauth2securityscheme.""" + code = "code" class BaseSecurityScheme(BaseModel): + """See https://www.w3.org/TR/wot-thing-description11/#securityscheme.""" + field_type: Optional[TypeDeclaration] = Field(None, alias="@type") description: Optional[Description] = None descriptions: Optional[Descriptions] = None @@ -331,6 +336,8 @@ class BaseSecurityScheme(BaseModel): class NoSecurityScheme(BaseSecurityScheme): + """See https://www.w3.org/TR/wot-thing-description11/#nosecurityscheme.""" + scheme: Literal[SecuritySchemeEnum.nosec] = SecuritySchemeEnum.nosec description: Optional[Description] = Field( default_factory=lambda: Description("No security") @@ -338,24 +345,34 @@ class NoSecurityScheme(BaseSecurityScheme): class NameAndIn(BaseModel): + """Fields used by various security schemas.""" + in_: Optional[In] = Field(None, alias="in") # for scheme=basic,digest,apikey,bearer name: Optional[str] = None # for scheme=basic,digest,apikey,bearer class BasicSecurityScheme(BaseSecurityScheme, NameAndIn): + """See https://www.w3.org/TR/wot-thing-description11/#basicsecurityscheme.""" + scheme: Literal[SecuritySchemeEnum.basic] = SecuritySchemeEnum.basic class DigestSecurityScheme(BaseSecurityScheme, NameAndIn): + """See https://www.w3.org/TR/wot-thing-description11/#digestsecurityscheme.""" + scheme: Literal[SecuritySchemeEnum.digest] = SecuritySchemeEnum.digest qop: Optional[Qop] = None # for scheme=digest class APISecurityScheme(BaseSecurityScheme, NameAndIn): + """See https://www.w3.org/TR/wot-thing-description11/#apisecurityscheme.""" + scheme: Literal[SecuritySchemeEnum.apikey] = SecuritySchemeEnum.apikey class BearerSecurityScheme(BaseSecurityScheme, NameAndIn): + """See https://www.w3.org/TR/wot-thing-description11/#bearersecurityscheme.""" + scheme: Literal[SecuritySchemeEnum.bearer] = SecuritySchemeEnum.bearer authorization: Optional[AnyUri] = None # for scheme=bearer,oauth2 alg: Optional[str] = None # for scheme=bearer @@ -363,11 +380,15 @@ class BearerSecurityScheme(BaseSecurityScheme, NameAndIn): class PskSecurityScheme(BaseSecurityScheme): + """See https://www.w3.org/TR/wot-thing-description11/#psksecurityscheme.""" + scheme: Literal[SecuritySchemeEnum.psk] = SecuritySchemeEnum.psk identity: Optional[str] = None # for scheme=psk class Oauth2SecurityScheme(BaseSecurityScheme): + """See https://www.w3.org/TR/wot-thing-description11/#oauth2securityscheme.""" + scheme: Literal[SecuritySchemeEnum.oauth2] = SecuritySchemeEnum.oauth2 authorization: Optional[AnyUri] = None # for scheme=bearer,oauth2 token: Optional[AnyUri] = None # for schema=oauth2 @@ -389,6 +410,8 @@ class Oauth2SecurityScheme(BaseSecurityScheme): class WotTdSchema16October2019(BaseModel): + """See https://www.w3.org/TR/wot-thing-description11/#thing.""" + model_config = ConfigDict(extra="allow") id: Optional[AnyUrl] = None diff --git a/src/labthings_fastapi/thing_description/validation.py b/src/labthings_fastapi/thing_description/validation.py index 381ba9dd..949d8a6d 100644 --- a/src/labthings_fastapi/thing_description/validation.py +++ b/src/labthings_fastapi/thing_description/validation.py @@ -1,6 +1,17 @@ +"""Validate a generated Thing Description against the W3C schema. + +We generate :ref:`wot_td` using `pydantic` models so there is a layer of +validation applied every time one is created. However, this module allows +the generated JSON document to be formally validated against the schema +in the W3C specification, as an additional check. + +See :ref:`wot_td` for a link to the specification in human-readable format. +""" + from importlib.resources import files import json import jsonschema +import jsonschema.exceptions from .. import thing_description import time import logging @@ -10,10 +21,18 @@ def validate_thing_description(td: dict) -> None: """Validate a Thing Description. This accepts a dictionary (usually generated from - `labthings_fastapi.thing_description.model.ThingDescription.model_dump()` + `labthings_fastapi.thing_description._model.ThingDescription.model_dump()` ) and validates it against the JSON schema for Thing Descriptions. This is obtained from the W3C's Thing Description repository on GitHub, URL in the file. + + No return value is provided, but a `~jsonschema.exceptions.ValidationError` + is raised if the schema is invalid. + + :param td: the Thing Description to be validated. + + :raise jsonschema.exceptions.ValidationError: if the Thing Description is + invalid. """ start = time.time() td_file = files(thing_description).joinpath("td-json-schema-validation.json") @@ -24,9 +43,12 @@ def validate_thing_description(td: dict) -> None: jsonschema.Draft7Validator.check_schema(schema) validated_schema = time.time() # Validate the TD dictionary - jsonschema.validate(instance=td, schema=schema) + try: + jsonschema.validate(instance=td, schema=schema) + except jsonschema.exceptions.ValidationError as e: + raise e validated_td = time.time() - logging.info( + logging.debug( f"Thing Description validated OK (schema load: {loaded_schema - start:.1f}s, " f"schema validation: {validated_schema - loaded_schema:.1f}s, TD validation: " f"{validated_td - validated_schema:.1f}s)" diff --git a/src/labthings_fastapi/types/__init__.py b/src/labthings_fastapi/types/__init__.py index e69de29b..b3ad59d7 100644 --- a/src/labthings_fastapi/types/__init__.py +++ b/src/labthings_fastapi/types/__init__.py @@ -0,0 +1,6 @@ +"""Types for use with LabThings. + +This module is intended to contain a range of types used by LabThings-FastAPI. +Currently, it only contains an annotated-type work-around for making `numpy` +arrays serialisable with `pydantic`. +""" diff --git a/src/labthings_fastapi/types/numpy.py b/src/labthings_fastapi/types/numpy.py index 1f184a5e..e8d067fb 100644 --- a/src/labthings_fastapi/types/numpy.py +++ b/src/labthings_fastapi/types/numpy.py @@ -1,15 +1,17 @@ -"""Basic support for numpy arrays in Pydantic models +"""Basic support for numpy arrays in Pydantic models. -We define a type alias `NDArray` which is a numpy array, annotated +We define a type alias `.NDArray` which is a numpy array, annotated to allow `pydantic` to convert it to and from JSON (as an array-of-arrays). -Usage: -``` -from labthings_fastapi.types.ndarray import NDArray +This should allow numpy arrays to be used without explicit conversion: -def double(arr: NDArray) -> NDArray: - return arr * 2 # arr is a numpy.ndarray -``` +.. code-block:: python + + from labthings_fastapi.types.ndarray import NDArray + + + def double(arr: NDArray) -> NDArray: + return arr * 2 # arr is a numpy.ndarray The implementation is not super elegant: it isn't recursive so has only been defined for up to 6d arrays. Specifying the dimensionality might be a nice @@ -51,20 +53,33 @@ def double(arr: NDArray) -> NDArray: class NestedListOfNumbersModel(RootModel): + """A RootModel describing a list-of-lists up to 7 deep. + + This is used to generate a JSONSchema description of a `numpy.ndarray` + serialised to a list. It is used in the annotated `.NDArray` type. + """ + root: NestedListOfNumbers def np_to_listoflists(arr: np.ndarray) -> NestedListOfNumbers: - """Convert a numpy array to a list of lists + """Convert a numpy array to a list of lists. NB this will not be quick! Large arrays will be much better serialised by dumping to base64 encoding or similar. + + :param arr: a `numpy.ndarray`. + :return: a nested list of numbers. """ return arr.tolist() # type: ignore[return-value] def listoflists_to_np(lol: Union[NestedListOfNumbers, np.ndarray]) -> np.ndarray: - """Convert a list of lists to a numpy array (or pass-through ndarrays)""" + """Convert a list of lists to a numpy array (or pass-through ndarrays). + + :param lol: a nested list of numbers. + :return: a `numpy.ndarray`. + """ return np.asarray(lol) @@ -77,10 +92,30 @@ def listoflists_to_np(lol: Union[NestedListOfNumbers, np.ndarray]) -> np.ndarray ), WithJsonSchema(NestedListOfNumbersModel.model_json_schema(), mode="validation"), ] +r"""An annotated type that enables `pydantic` to handle `numpy.ndarray`\ . + +`.NDArray` "validates" `numpy.ndarray` objects by converting a nested list of +numbers into an `numpy.ndarray` using `numpy.asarray`\ . Similarly, it calls +`numpy.ndarray.tolist` to convert the array back into a serialisable structure. + +The JSON Schema representation is a nested list up to 7 deep, which is cumbersome +but correct. + +In the future it would be good to replace this type with several types of +different, specified dimensionality. That would make for much less horrible +:ref:`wot_td` representations, as well as giving useful information about the datatype +returned. +""" def denumpify(v: Any) -> Any: - """Convert any numpy array in a dict into a list""" + """Convert any numpy array in a dict into a list. + + :param v: the data to convert, may be a mapping, sequence, or other. + + :return: the input datastructure, with all `numpy.ndarray` objects + converted to lists. + """ if isinstance(v, np.ndarray): return v.tolist() elif isinstance(v, Mapping): @@ -92,10 +127,23 @@ def denumpify(v: Any) -> Any: def denumpify_serializer(v: Any, nxt: SerializerFunctionWrapHandler) -> Any: - """A Pydantic wrap serializer to denumpify mappings before serialization""" + """Denumpify mappings before serialization. + + This is intended for use as a "wrap serialiser" in `pydantic`, and + will remove `numpy.ndarray` objects from a data structure using + `.denumpify`. This should allow dicts containing `numpy.ndarray` + objects to be serialised to JSON. + + :param v: input data, of any type. + :param nxt: the next serialiser, see `pydantic` docs. + + :return: the data structure with `numpy.ndarray` objects removed. + """ return nxt(denumpify(v)) class DenumpifyingDict(RootModel): + """A `pydantic` model for a dictionary that converts arrays to lists.""" + root: Annotated[Mapping, WrapSerializer(denumpify_serializer)] model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/src/labthings_fastapi/utilities/__init__.py b/src/labthings_fastapi/utilities/__init__.py index a9807ffb..4b5008a4 100644 --- a/src/labthings_fastapi/utilities/__init__.py +++ b/src/labthings_fastapi/utilities/__init__.py @@ -1,3 +1,5 @@ +"""Utility functions used by LabThings-FastAPI.""" + from __future__ import annotations from typing import Any, Dict, Iterable, TYPE_CHECKING, Optional from weakref import WeakSet @@ -11,16 +13,29 @@ def class_attributes(obj: Any) -> Iterable[tuple[str, Any]]: - """A list of all the attributes of an object's class""" + """List all the attributes of an object's class. + + This function gets all class attributes, including inherited ones. + It is used to obtain the various descriptors used to represent + properties and actions. It calls `.attributes` on ``obj.__class__``. + + :param obj: The instance, usually a `.Thing` instance. + + :yield: tuples of ``(name, value)`` giving each attribute of the class. + """ cls = obj.__class__ - for name in dir(cls): - if name.startswith("__"): - continue - yield name, getattr(cls, name) + yield from attributes(cls) def attributes(cls: Any) -> Iterable[tuple[str, Any]]: - """A list of all the attributes of an object not starting with `__`""" + """List all the attributes of an object not starting with `__`. + + :param cls: The object whose attributes we are listing. This may be + a class, because classes are objects too. + + :yield: tuples of ``(name, value)`` giving each attribute and its + value. + """ for name in dir(cls): if name.startswith("__"): continue @@ -32,29 +47,71 @@ def attributes(cls: Any) -> Iterable[tuple[str, Any]]: @dataclass(config=ConfigDict(arbitrary_types_allowed=True)) class LabThingsObjectData: + r"""Data used by LabThings, stored on each `.Thing`. + + This `pydantic.dataclass` groups together some properties used + by LabThings descriptors, to avoid cluttering the namespace of the + `.Thing` subclass on which they are defined. + """ + property_observers: Dict[str, WeakSet] = Field(default_factory=dict) + r"""The observers added to each property. + + Keys are property names, values are weak sets used by `.ThingProperty`\ . + """ action_observers: Dict[str, WeakSet] = Field(default_factory=dict) + r"""The observers added to each action. + + Keys are action names, values are weak sets used by + `.ActionDescriptor`\ . + """ def labthings_data(obj: Thing) -> LabThingsObjectData: - """Get (or create) a dictionary for LabThings properties""" + """Get (or create) a dictionary for LabThings properties. + + Ensure there is a `.LabThingsObjectData` dataclass attached to + a particular `.Thing`, and return it. + + :param obj: The `.Thing` we are looking for the dataclass on. + + :return: a `.LabThingsObjectData` instance attached to ``obj``. + """ if LABTHINGS_DICT_KEY not in obj.__dict__: obj.__dict__[LABTHINGS_DICT_KEY] = LabThingsObjectData() return obj.__dict__[LABTHINGS_DICT_KEY] def get_blocking_portal(obj: Thing) -> Optional[BlockingPortal]: - """Retrieve a blocking portal from a Thing""" + """Retrieve a blocking portal from a Thing. + + See :ref:`concurrency` for more details. + + When a `.Thing` is attached to a `.ThingServer` and the `.ThingServer` + is started, it sets an attribute on each `.Thing` to allow it to + access an `anyio.from_thread.BlockingPortal`. This allows threaded + code to call async code. + + This function retrieves the blocking portal from a `.Thing`. + + :param obj: the `.Thing` on which we are looking for the portal. + + :return: the blocking portal. + """ return obj._labthings_blocking_portal def wrap_plain_types_in_rootmodel(model: type) -> type[BaseModel]: """Ensure a type is a subclass of BaseModel. - If a `BaseModel` subclass is passed to this function, we will pass it - through unchanged. Otherwise, we wrap the type in a RootModel. + If a `pydantic.BaseModel` subclass is passed to this function, we will pass it + through unchanged. Otherwise, we wrap the type in a `pydantic.RootModel`. In the future, we may explicitly check that the argument is a type and not a model instance. + + :param model: A Python type or `pydantic` model. + + :return: A `pydantic` model, wrapping Python types in a ``RootModel`` if needed. """ try: # This needs to be a `try` as basic types are not classes assert issubclass(model, BaseModel) @@ -64,16 +121,23 @@ def wrap_plain_types_in_rootmodel(model: type) -> type[BaseModel]: def model_to_dict(model: Optional[BaseModel]) -> Dict[str, Any]: - """Convert a pydantic model to a dictionary + """Convert a pydantic model to a dictionary, non-recursively. We convert only the top level model, i.e. we do not recurse into submodels. This is important to avoid serialising Blob objects in action inputs. This function returns `dict(model)`, with exceptions for the case of `None` - (converted to an empty dictionary) and `RootModel`s (checked to see if they - correspond to empty input). + (converted to an empty dictionary) and `pydantic.RootModel` (checked to see + if they correspond to empty input). + + If `pydantic.RootModel` with non-empty input is allowed, this function will + need to be updated to handle them. + + :param model: A Pydantic model (usually the input of an action). + + :return: A dictionary with string keys, which are the fields of the model. + This should be suitable for using as ``**kwargs`` to an action. - If RootModels with non-empty input are allowed, this function will need to - be updated to handle them. + :raise ValueError: if we are given a root model that isn't empty. """ if model is None: return {} diff --git a/src/labthings_fastapi/utilities/introspection.py b/src/labthings_fastapi/utilities/introspection.py index 38477515..8cbeaf6a 100644 --- a/src/labthings_fastapi/utilities/introspection.py +++ b/src/labthings_fastapi/utilities/introspection.py @@ -1,5 +1,4 @@ -""" -A collection of utility functions to analyse types and metadata +"""A collection of utility functions to analyse types and metadata. Many parts of LabThings require us to use type annotations to generate schemas/validation/documentation. This is done using @@ -20,18 +19,32 @@ class EmptyObject(BaseModel): + """A model representing an object with no required keys.""" + model_config = ConfigDict(extra="allow") class StrictEmptyObject(EmptyObject): + """A model representing an object that must have no keys.""" + model_config = ConfigDict(extra="forbid") class EmptyInput(RootModel): + """Represent the input of an action that has no required parameters. + + This may be either a dictionary or ``None``. + """ + root: Optional[EmptyObject] = None class StrictEmptyInput(EmptyInput): + """Represent the input of an action that never takes parameters. + + This may be either an empty dictionary or ``None``. + """ + root: Optional[StrictEmptyObject] = None @@ -45,21 +58,27 @@ def input_model_from_signature( This is deliberately quite a lot more basic than `pydantic.decorator.ValidatedFunction` because it is designed to handle JSON input. That means that we don't want positional - arguments, unless there's exactly one (in which case we have a - single value, not an object, and this may or may not be supported). + arguments. - This will fail for position-only arguments, though that may change - in the future. + .. note:: + LabThings-FastAPI does not currently support actions that take + positional arguments, because this does not convert nicely into + JSONSchema or Thing Description documents (see :ref:`wot_td`). + + :param func: the function to analyse. :param remove_first_positional_arg: Remove the first argument from the model (this is appropriate for methods, as the first argument, self, is baked in when it's called, but is present in the signature). :param ignore: Ignore arguments that have the specified name. This is useful for e.g. dependencies that are injected by LabThings. - :returns: A pydantic model class describing the input parameters - TODO: deal with (or exclude) functions with a single positional parameter + :return: A pydantic model class describing the input parameters + + :raise TypeError: if positional arguments are used: this is not supported. + :raise ValueError: if ``remove_first_positional_arg`` is true but there + is no initial positional argument. """ parameters: OrderedDict[str, Parameter] = OrderedDict(signature(func).parameters) if remove_first_positional_arg: @@ -109,29 +128,29 @@ def input_model_from_signature( return model -def function_dependencies( - func: Callable, dependency_types: Sequence[Type] -) -> Dict[str, tuple[type, type]]: - """Determine whether a function's arguments require dependencies +def fastapi_dependency_params(func: Callable) -> Sequence[Parameter]: + """Find the arguments of a function that are FastAPI dependencies. + + This allows us to "pass through" the full power of the FastAPI dependency + injection system to thing actions. Any function parameter that has a + type hint annotated with `fastapi.Depends` will be treated as a + dependency, and thus be supplied automatically when it is called over + HTTP. See :ref:`dependencies` for an overview. - The return value maps argument names to a tuple of (type, full_type) - where `full_type` is the annotation without simplification, i.e. - it will include the contents of any Annotated objects. - """ - type_hints = get_type_hints(func, include_extras=False) - full_type_hints = get_type_hints(func, include_extras=True) - return { - name: (type_, full_type_hints[name]) - for name, type_ in type_hints.items() - if type_ in dependency_types - } + We give special treatment to dependency parameters, as they must not + appear in the input model, and they must be supplied by the + `.DirectThingClient` wrapper to make the signature identical to that + of the `.ThingClient` over HTTP. + .. note:: -def fastapi_dependency_params(func: Callable) -> Sequence[Parameter]: - """Find the arguments of a function that are FastAPI dependencies + Path and query parameters are ignored. These should not be used as action + parameters, and will most likely raise an error when the `.Thing` is + added to FastAPI. - This allows us to "pass through" the full power of the FastAPI dependency - injection system to thing actions. + :param func: a function to inspect. + + :return: a list of parameter objects that are annotated as dependencies. """ # TODO: this currently ignores path parameters sig = get_typed_signature(func) @@ -149,7 +168,12 @@ def fastapi_dependency_params(func: Callable) -> Sequence[Parameter]: def return_type(func: Callable) -> Type: - """Determine the return type of a function.""" + """Determine the return type of a function. + + :param func: a function to inspect + + :return: the return type of the function. + """ sig = inspect.signature(func) if sig.return_annotation == inspect.Signature.empty: return Any # type: ignore[return-value] @@ -160,23 +184,24 @@ def return_type(func: Callable) -> Type: return type_hints["return"] -def get_docstring(obj: Any, remove_summary=False) -> Optional[str]: - """Return the docstring of an object +def get_docstring(obj: Any, remove_summary: bool = False) -> Optional[str]: + """Return the docstring of an object. + + Get the docstring of an object, optionally removing the initial "summary" + line. - If `remove_newlines` is `True` (default), newlines are removed from the string. If `remove_summary` is `True` (not default), and the docstring's second line is blank, the first two lines are removed. If the docstring follows the convention of a one-line summary, a blank line, and a description, this will get just the description. - If `remove_newlines` is `False`, the docstring is processed by + The docstring is processed by `inspect.cleandoc()` to remove whitespace from the start of each line. - :param obj: Any Python object - :param remove_newlines: bool (Default value = True) - :param remove_summary: bool (Default value = False) - :returns: str: Object docstring + :param obj: Any Python object. + :param remove_summary: whether to remove the summary line, if present. + :return: The object's docstring. """ ds = obj.__doc__ if not ds: @@ -189,11 +214,10 @@ def get_docstring(obj: Any, remove_summary=False) -> Optional[str]: def get_summary(obj: Any) -> Optional[str]: - """Return the first line of the dosctring of an object + """Return the first line of the dosctring of an object. :param obj: Any Python object - :returns: str: First line of object docstring - + :return: First line of object docstring, or ``None``. """ docs = get_docstring(obj) if docs: diff --git a/src/labthings_fastapi/utilities/object_reference_to_object.py b/src/labthings_fastapi/utilities/object_reference_to_object.py index f7b556ed..d95e543c 100644 --- a/src/labthings_fastapi/utilities/object_reference_to_object.py +++ b/src/labthings_fastapi/utilities/object_reference_to_object.py @@ -1,8 +1,11 @@ +"""Load objects from object references.""" + import importlib +from typing import Any -def object_reference_to_object(object_reference: str): - """Convert a string reference to an object +def object_reference_to_object(object_reference: str) -> Any: + """Convert a string reference to an object. This is taken from: https://packaging.python.org/en/latest/specifications/entry-points/ @@ -10,6 +13,12 @@ def object_reference_to_object(object_reference: str): The format of the string is `module_name:qualname` where `qualname` is the fully qualified name of the object within the module. This is the same format used by entrypoints` in `setup.py` files. + + :param object_reference: a string referencing a Python object to import. + + :return: the Python object. + + :raise ImportError: if the referenced object cannot be found or imported. """ modname, qualname_separator, qualname = object_reference.partition(":") obj = importlib.import_module(modname) diff --git a/src/labthings_fastapi/websockets.py b/src/labthings_fastapi/websockets.py index 8c1d4c96..74a0f783 100644 --- a/src/labthings_fastapi/websockets.py +++ b/src/labthings_fastapi/websockets.py @@ -1,5 +1,4 @@ -""" -Handle notification of events, property, and action status changes +"""Handle notification of events, property, and action status changes. There are several kinds of "event" in the WoT vocabulary, not all of which are called Event, which is why this module is called `notifications`. @@ -34,11 +33,15 @@ async def relay_notifications_to_websocket( websocket: WebSocket, receive_stream: ObjectReceiveStream ) -> None: - """Relay objects from a stream to a websocket as JSON + """Relay objects from a stream to a websocket as JSON. - Interaction affordances (events, actions) that we've registered with will + :ref:`wot_affordances` (events, actions) that we've registered with will post messages to the queue: this function takes those messages from the queue and passes them to the websocket. + + :param websocket: the WebSocket we are communicating over. + :param receive_stream: an `anyio.abc.ObjectReceiveStream` that will + yield objects that we send over the websocket. """ async with receive_stream: async for item in receive_stream: @@ -48,10 +51,17 @@ async def relay_notifications_to_websocket( async def process_messages_from_websocket( websocket: WebSocket, send_stream: ObjectSendStream, thing: Thing ) -> None: - """Process messages received from a websocket + r"""Process messages received from a websocket. Currently, this will allow us to observe properties, by registering (or de-registering) for those properties. + + :param websocket: the WebSocket we are communicating over. + :param send_stream: an `anyio.abc.ObjectSendStream` that we + use to register for events, i.e. data sent to that stream will + be sent through this websocket, by `.relay_notifications_to_websocket`\ . + :param thing: the `.Thing` we are attached to. The websocket is specific to + one `.Thing`, and this is it. """ while True: try: @@ -70,7 +80,15 @@ async def process_messages_from_websocket( async def websocket_endpoint(thing: Thing, websocket: WebSocket) -> None: - """Handle communication to a client via websocket""" + r"""Handle communication to a client via websocket. + + This function handles a websocket connection to a `.Thing`\ 's websocket + endpoint. It can add observers to properties and actions, and will forward + notifications from the property or action back to the websocket. + + :param thing: the `.Thing` the websocket is attached to. + :param websocket: the web socket that has been created. + """ await websocket.accept() send_stream, receive_stream = create_memory_object_stream[dict]() async with create_task_group() as tg: diff --git a/tests/test_td_dataschema_generation.py b/tests/test_td_dataschema_generation.py index 733e65af..86f807f6 100644 --- a/tests/test_td_dataschema_generation.py +++ b/tests/test_td_dataschema_generation.py @@ -4,7 +4,7 @@ import json from pydantic import BaseModel from typing import Optional -from labthings_fastapi.thing_description.model import DataSchema +from labthings_fastapi.thing_description._model import DataSchema def ds_json_dict(ds: DataSchema) -> dict: diff --git a/tests/test_thing_dependencies.py b/tests/test_thing_dependencies.py index 729e0f2f..1279cd37 100644 --- a/tests/test_thing_dependencies.py +++ b/tests/test_thing_dependencies.py @@ -152,7 +152,7 @@ def action_four(self, thing_two: ThingTwoDepNoActions) -> str: def action_five(self, thing_two: ThingTwoDep) -> str: return thing_two.action_two() - with pytest.raises(ValueError): + with pytest.raises(lt.client.in_server.DependencyNameClashError): lt.deps.direct_thing_client_dependency(ThingFour, "/thing_four/")