From 43127830cccc4219205050ec2b2752c2b365e4c2 Mon Sep 17 00:00:00 2001 From: Stanley Kudrow Date: Fri, 20 Jun 2025 10:45:35 +0300 Subject: [PATCH 1/6] rebase after the PR #219 --- CHANGELOG.md | 2 ++ src/aiofiles/base.py | 37 +++++++++++++++++------- src/aiofiles/os.py | 65 +++++++++++++++++++++--------------------- src/aiofiles/ospath.py | 26 ++++++++--------- 4 files changed, 74 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b327ffb..88278e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## 25.1.0 (UNRELEASED) +- Use `asyncio.to_thread` in the `wrap` asynchroniser + [#213](https://github.com/Tinche/aiofiles/pull/213) - Switch to [uv](https://docs.astral.sh/uv/) + add Python v3.14 support. ([#219](https://github.com/Tinche/aiofiles/pull/219)) - Add `ruff` formatter and linter. diff --git a/src/aiofiles/base.py b/src/aiofiles/base.py index ef1f81d..b3c1ae0 100644 --- a/src/aiofiles/base.py +++ b/src/aiofiles/base.py @@ -1,18 +1,35 @@ -from asyncio import get_running_loop -from collections.abc import Awaitable +from asyncio import get_running_loop, to_thread +from collections.abc import Awaitable, Callable, Coroutine from contextlib import AbstractAsyncContextManager -from functools import partial, wraps +from functools import wraps +from warnings import warn -def wrap(func): +def to_coro(func: Callable) -> Callable: + """Converts the routine `func` into a coroutine. + + The returned coroutine function runs the decorated function + in a separate thread. + + Args: + func: A routine (regular function). + + Returns: + A coroutine function. + """ + @wraps(func) - async def run(*args, loop=None, executor=None, **kwargs): - if loop is None: - loop = get_running_loop() - pfunc = partial(func, *args, **kwargs) - return await loop.run_in_executor(executor, pfunc) + async def _wrapper(*args, **kwargs) -> Coroutine: + return await to_thread(func, *args, **kwargs) + + return _wrapper + - return run +def wrap(func: Callable) -> Callable: + warn( + "scheduled to removal, consider using to_coro", DeprecationWarning, stacklevel=1 + ) + return to_coro(func) class AsyncBase: diff --git a/src/aiofiles/os.py b/src/aiofiles/os.py index 153d65d..a998d0b 100644 --- a/src/aiofiles/os.py +++ b/src/aiofiles/os.py @@ -3,59 +3,58 @@ import os from . import ospath as path -from .base import wrap +from .base import to_coro __all__ = [ - "path", - "stat", + "access", + "getcwd", + "listdir", + "makedirs", + "mkdir", + "readlink", + "remove", + "removedirs", "rename", "renames", "replace", - "remove", - "unlink", - "mkdir", - "makedirs", "rmdir", - "removedirs", - "symlink", - "readlink", - "listdir", + "path", "scandir", - "access", - "wrap", - "getcwd", + "stat", + "symlink", + "unlink", ] -access = wrap(os.access) +access = to_coro(os.access) -getcwd = wrap(os.getcwd) +getcwd = to_coro(os.getcwd) -listdir = wrap(os.listdir) +listdir = to_coro(os.listdir) -makedirs = wrap(os.makedirs) -mkdir = wrap(os.mkdir) +makedirs = to_coro(os.makedirs) +mkdir = to_coro(os.mkdir) -readlink = wrap(os.readlink) -remove = wrap(os.remove) -removedirs = wrap(os.removedirs) -rename = wrap(os.rename) -renames = wrap(os.renames) -replace = wrap(os.replace) -rmdir = wrap(os.rmdir) +readlink = to_coro(os.readlink) +remove = to_coro(os.remove) +removedirs = to_coro(os.removedirs) +rename = to_coro(os.rename) +renames = to_coro(os.renames) +replace = to_coro(os.replace) +rmdir = to_coro(os.rmdir) -scandir = wrap(os.scandir) -stat = wrap(os.stat) -symlink = wrap(os.symlink) +scandir = to_coro(os.scandir) +stat = to_coro(os.stat) +symlink = to_coro(os.symlink) -unlink = wrap(os.unlink) +unlink = to_coro(os.unlink) if hasattr(os, "link"): __all__ += ["link"] - link = wrap(os.link) + link = to_coro(os.link) if hasattr(os, "sendfile"): __all__ += ["sendfile"] - sendfile = wrap(os.sendfile) + sendfile = to_coro(os.sendfile) if hasattr(os, "statvfs"): __all__ += ["statvfs"] - statvfs = wrap(os.statvfs) + statvfs = to_coro(os.statvfs) diff --git a/src/aiofiles/ospath.py b/src/aiofiles/ospath.py index f47f150..b879c79 100644 --- a/src/aiofiles/ospath.py +++ b/src/aiofiles/ospath.py @@ -2,7 +2,7 @@ from os import path -from .base import wrap +from .base import to_coro __all__ = [ "abspath", @@ -19,19 +19,19 @@ "sameopenfile", ] -abspath = wrap(path.abspath) +abspath = to_coro(path.abspath) -getatime = wrap(path.getatime) -getctime = wrap(path.getctime) -getmtime = wrap(path.getmtime) -getsize = wrap(path.getsize) +getatime = to_coro(path.getatime) +getctime = to_coro(path.getctime) +getmtime = to_coro(path.getmtime) +getsize = to_coro(path.getsize) -exists = wrap(path.exists) +exists = to_coro(path.exists) -isdir = wrap(path.isdir) -isfile = wrap(path.isfile) -islink = wrap(path.islink) -ismount = wrap(path.ismount) +isdir = to_coro(path.isdir) +isfile = to_coro(path.isfile) +islink = to_coro(path.islink) +ismount = to_coro(path.ismount) -samefile = wrap(path.samefile) -sameopenfile = wrap(path.sameopenfile) +samefile = to_coro(path.samefile) +sameopenfile = to_coro(path.sameopenfile) From 74350869c1fa07ec491654ea24bcf41de5e861bf Mon Sep 17 00:00:00 2001 From: Stanley Kudrow Date: Sat, 21 Jun 2025 14:17:13 +0300 Subject: [PATCH 2/6] update the #213 record in the CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88278e1..8989c18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 25.1.0 (UNRELEASED) -- Use `asyncio.to_thread` in the `wrap` asynchroniser +- Use `asyncio.to_thread` in the `to_coro` decorator that deprecates the `wrap` decorator [#213](https://github.com/Tinche/aiofiles/pull/213) - Switch to [uv](https://docs.astral.sh/uv/) + add Python v3.14 support. ([#219](https://github.com/Tinche/aiofiles/pull/219)) From deada0d4b85214c9747da1a02a62fd1dbc126086 Mon Sep 17 00:00:00 2001 From: Stanley Kudrow Date: Sat, 21 Jun 2025 14:27:16 +0300 Subject: [PATCH 3/6] remove the 'run_prefix' from the Justfile --- Justfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Justfile b/Justfile index ccae1b8..5039573 100644 --- a/Justfile +++ b/Justfile @@ -1,19 +1,19 @@ +src_dir := "src" tests_dir := "tests" -code_dirs := "src" + " " + tests_dir -run_prefix := if env_var_or_default("VIRTUAL_ENV", "") == "" { "uv run " } else { "" } +code_dirs := src_dir + " " + tests_dir check: - {{ run_prefix }}ruff format --check {{ code_dirs }} - {{ run_prefix }}ruff check {{ code_dirs }} + ruff format --check {{ code_dirs }} + ruff check {{ code_dirs }} coverage: - {{ run_prefix }}coverage run -m pytest {{ tests_dir }} + coverage run -m pytest {{ tests_dir }} format: - {{ run_prefix }}ruff format {{ code_dirs }} + ruff format {{ code_dirs }} lint: format - {{ run_prefix }}ruff check --fix {{ code_dirs }} + ruff check --fix {{ code_dirs }} test: - {{ run_prefix }}pytest -x --ff {{ tests_dir }} \ No newline at end of file + pytest -x --ff {{ tests_dir }} \ No newline at end of file From a99011412ea489756ec7c14d4f70052ca52825d8 Mon Sep 17 00:00:00 2001 From: Stanley Kudrow Date: Sat, 21 Jun 2025 14:29:55 +0300 Subject: [PATCH 4/6] add pytest addopts in the pyproject.toml --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 3efac5f..67dc815 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,10 @@ source = [ [tool.pytest.ini_options] minversion = "8.3" +addopts = [ + "-x", + "--ff", +] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" From 59fc35a8b16153e85ea06fc46edb5f0a0e9ad435 Mon Sep 17 00:00:00 2001 From: Stanley Kudrow Date: Sat, 21 Jun 2025 14:30:55 +0300 Subject: [PATCH 5/6] remove opts from the Justfile test rule --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 5039573..0fe290e 100644 --- a/Justfile +++ b/Justfile @@ -16,4 +16,4 @@ lint: format ruff check --fix {{ code_dirs }} test: - pytest -x --ff {{ tests_dir }} \ No newline at end of file + pytest {{ tests_dir }} From a69548b68f3de2d54b56d469d22a648dd0d8582b Mon Sep 17 00:00:00 2001 From: Stanley Kudrow Date: Sat, 21 Jun 2025 14:35:53 +0300 Subject: [PATCH 6/6] rebase after the PR #213 --- CHANGELOG.md | 1 + pyproject.toml | 1 + src/aiofiles/base.py | 47 ++++++++++++++++++++++++++-- src/aiofiles/os.py | 5 ++- tests/test_os.py | 73 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 123 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8989c18..3bf7d78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 25.1.0 (UNRELEASED) +- Add `to_agen` generator asynchroniser tool with the `os.walk` asynchronous analogue introduced - Use `asyncio.to_thread` in the `to_coro` decorator that deprecates the `wrap` decorator [#213](https://github.com/Tinche/aiofiles/pull/213) - Switch to [uv](https://docs.astral.sh/uv/) + add Python v3.14 support. diff --git a/pyproject.toml b/pyproject.toml index 67dc815..65f67b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ minversion = "8.3" addopts = [ "-x", "--ff", + "--maxfail=1" ] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" diff --git a/src/aiofiles/base.py b/src/aiofiles/base.py index b3c1ae0..4b88386 100644 --- a/src/aiofiles/base.py +++ b/src/aiofiles/base.py @@ -1,10 +1,52 @@ from asyncio import get_running_loop, to_thread -from collections.abc import Awaitable, Callable, Coroutine +from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine from contextlib import AbstractAsyncContextManager -from functools import wraps +from functools import partial, wraps +from queue import Empty, Queue from warnings import warn +def to_agen(func: Callable) -> AsyncIterator: + """Converts the routine `func` into an async generator function. + + Args: + func: A generator function. + + Returns: + An asynchronous generator function. + """ + + @wraps(func) + async def _wrapper(*args, **kwargs) -> AsyncIterator: + def _generate(q: Queue): + nonlocal is_over + try: + for row in func(*args, **kwargs): + q.put_nowait(row) + finally: + is_over = True + + loop = get_running_loop() + queue: Queue = Queue() + + is_over = False + + gen = partial(_generate, q=queue) + loop.run_in_executor(None, gen) + while True: + if is_over and queue.empty(): + break + try: + item = queue.get_nowait() + queue.task_done() + yield item + except Empty: + pass + queue.join() + + return _wrapper + + def to_coro(func: Callable) -> Callable: """Converts the routine `func` into a coroutine. @@ -43,7 +85,6 @@ def _loop(self): return self._ref_loop or get_running_loop() def __aiter__(self): - """We are our own iterator.""" return self def __repr__(self): diff --git a/src/aiofiles/os.py b/src/aiofiles/os.py index a998d0b..79285bb 100644 --- a/src/aiofiles/os.py +++ b/src/aiofiles/os.py @@ -3,7 +3,7 @@ import os from . import ospath as path -from .base import to_coro +from .base import to_agen, to_coro __all__ = [ "access", @@ -23,6 +23,7 @@ "stat", "symlink", "unlink", + "walk", ] access = to_coro(os.access) @@ -48,6 +49,8 @@ unlink = to_coro(os.unlink) +walk = to_agen(os.walk) + if hasattr(os, "link"): __all__ += ["link"] diff --git a/tests/test_os.py b/tests/test_os.py index 2c75bec..72d94eb 100644 --- a/tests/test_os.py +++ b/tests/test_os.py @@ -3,9 +3,11 @@ import asyncio import os import platform +from collections import defaultdict from os import stat from os.path import dirname, exists, isdir, join from pathlib import Path +from typing import Optional import pytest @@ -498,3 +500,74 @@ async def test_abspath(): abs_filename = join(dirname(__file__), "resources", "test_file1.txt") result = await aiofiles.os.path.abspath(relative_filename) assert result == abs_filename + + +def _act_on_error(oserror): + print(f"aiofiles.os.walk onerror handler: {oserror}") + + +@pytest.mark.parametrize( + ("top", "topdown", "onerror", "followlinks"), + [ + ("non-existent", True, _act_on_error, False), + ("README.md", True, None, False), + ("README.md", False, _act_on_error, False), + ("./src", True, _act_on_error, False), + ("./src", False, _act_on_error, False), + ("./tests", True, None, True), + ("./tests", False, None, True), + ], +) +async def test_walk(top, topdown, onerror, followlinks): + result = [ # noqa: C416 + row + async for row in aiofiles.os.walk( + top=top, topdown=topdown, onerror=onerror, followlinks=followlinks + ) + ] + answer = list( + os.walk(top=top, topdown=topdown, onerror=onerror, followlinks=followlinks) + ) + + assert result == answer + + +@pytest.mark.parametrize( + "top_path", + [ + "c3Po-ar2D2", # does not mean to exist + "./.github", + "./src", + "./tests", + # ".", # it is long + ], +) +async def test_walk_non_blocking(top_path: str): + async def _test_walk( + top: str, key: str, storage: defaultdict[str, list] + ) -> Optional[bool]: + non_blocking = None + async for datum in aiofiles.os.walk(top=top): + storage[key].append(datum) + # checking the statistics (common shareable resource) for other tasks + other_keys = set(storage) - {key} + if not non_blocking: + non_blocking = any(storage[other_key] for other_key in other_keys) + # without the `asyncio.sleep(0)` the `async for` may block + await asyncio.sleep(0) + + return non_blocking + + keys = {f"w_{i}" for i in range(2)} + storage = defaultdict(list) + + tasks: list[asyncio.Task] = [ + asyncio.create_task(_test_walk(top=top_path, key=key, storage=storage)) + for key in keys + ] + results = await asyncio.gather(*tasks) + + # Asserting that each task does not block the event loop. + # Empty results are not assumed to be non-blocking by default. + if statuses := (r for r in results if r is not None): + assert all(statuses)