From 68509eb9b651d20ebeb62c598fc9644fb533c19d Mon Sep 17 00:00:00 2001 From: chemicstry Date: Wed, 30 Apr 2025 00:42:56 +0300 Subject: [PATCH 1/7] Impllement basic file client operations --- yakut/__init__.py | 2 +- yakut/cmd/file_client/__init__.py | 6 + yakut/cmd/file_client/_cmd.py | 303 +++++++++++++++++++++++++++ yakut/cmd/file_client/_file_error.py | 14 ++ yakut/cmd/file_client/_list_files.py | 181 ++++++++++++++++ yakut/main.py | 1 + 6 files changed, 506 insertions(+), 1 deletion(-) create mode 100644 yakut/cmd/file_client/__init__.py create mode 100644 yakut/cmd/file_client/_cmd.py create mode 100644 yakut/cmd/file_client/_file_error.py create mode 100644 yakut/cmd/file_client/_list_files.py diff --git a/yakut/__init__.py b/yakut/__init__.py index 1e6696c..693c467 100644 --- a/yakut/__init__.py +++ b/yakut/__init__.py @@ -27,6 +27,6 @@ def _read_package_file(name: str) -> str: __copyright__ = f"Copyright (c) 2020 {__author__} <{__email__}>" __license__ = "MIT" -from .main import main as main, subcommand as subcommand, Purser as Purser, pass_purser as pass_purser +from .main import main as main, subcommand as subcommand, commandgroup as commandgroup, Purser as Purser, pass_purser as pass_purser from .main import asynchronous as asynchronous, get_logger as get_logger from . import cmd as cmd diff --git a/yakut/cmd/file_client/__init__.py b/yakut/cmd/file_client/__init__.py new file mode 100644 index 0000000..b0ce018 --- /dev/null +++ b/yakut/cmd/file_client/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2021 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +import enum +from ._cmd import file_client as file_client diff --git a/yakut/cmd/file_client/_cmd.py b/yakut/cmd/file_client/_cmd.py new file mode 100644 index 0000000..122d772 --- /dev/null +++ b/yakut/cmd/file_client/_cmd.py @@ -0,0 +1,303 @@ +# Copyright (c) 2021 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import sys +from typing import TYPE_CHECKING +from pathlib import Path, PurePosixPath +import click +import pycyphal +import yakut +from yakut.int_set_parser import parse_int_set +from yakut.param.formatter import FormatterHints +from yakut.ui import ProgressReporter, show_error, show_warning +from yakut.util import EXIT_CODE_UNSUCCESSFUL +from ._list_files import list_files +from ._file_error import FileError + +if TYPE_CHECKING: + import pycyphal.application # pylint: disable=ungrouped-imports + import pycyphal.application.file # pylint: disable=ungrouped-imports + +_logger = yakut.get_logger(__name__) + +@yakut.commandgroup(aliases="fcli") +@click.pass_context +@yakut.pass_purser +def file_client(purser: yakut.Purser, cmd: str): + """Main CLI group.""" + pass + +@file_client.command() +@click.argument("node_ids", type=parse_int_set) +@click.argument("path", default="") +@click.option( + "--timeout", + "-T", + type=float, + default=pycyphal.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, + show_default=True, + metavar="SECONDS", + help="Service response timeout.", +) +@click.option( + "--optional-service", + "-s", + is_flag=True, + help=""" +Ignore nodes that fail to respond to the first RPC-service request instead of reporting an error +assuming that the register service is not supported. +If a node responded at least once it is assumed to support the service and any future timeout +will be always treated as an error. +""", +) +@click.option( + "--get-info", + "-i", + is_flag=True, + help="Also request GetInfo for each file.", +) +@yakut.pass_purser +@yakut.asynchronous(interrupted_ok=True) +async def file_list( + purser: yakut.Purser, + node_ids: set[int] | int, + path: Path, + timeout: float, + optional_service: bool, + get_info: bool, +) -> None: + _logger.debug("node_ids=%r, path=%r, timeout=%r", node_ids, path, timeout) + node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] + assert isinstance(node_ids_list, list) and all(isinstance(x, int) for x in node_ids_list) + formatter = purser.make_formatter(FormatterHints(single_document=True)) + with purser.get_node("file_client_list", allow_anonymous=False) as node: + with ProgressReporter() as prog: + result = await list_files( + node, + prog, + node_ids_list, + path, + optional_service=optional_service, + get_info=get_info, + timeout=timeout, + ) + # The node is no longer needed. + for msg in result.errors: + show_error(msg) + for msg in result.warnings: + show_warning(msg) + final = result.files_per_node if not isinstance(node_ids, int) else result.files_per_node[node_ids] + sys.stdout.write(formatter(final)) + sys.stdout.flush() + + return EXIT_CODE_UNSUCCESSFUL if result.errors else 0 + +@file_client.command() +@click.argument("node_ids", type=parse_int_set) +@click.argument("path", default="") +@click.option( + "--timeout", + "-T", + type=float, + default=pycyphal.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, + show_default=True, + metavar="SECONDS", + help="Service response timeout.", +) +@yakut.pass_purser +@yakut.asynchronous(interrupted_ok=True) +async def file_remove( + purser: yakut.Purser, + node_ids: set[int] | int, + path: Path, + timeout: float, +) -> None: + try: + from uavcan.file import Path_2_0 + from uavcan.file import Error_1_0 + from uavcan.file import Modify_1_1 + except ImportError as ex: + from yakut.cmd.compile import make_usage_suggestion + + raise click.ClickException(make_usage_suggestion(ex.name)) from None + + _logger.debug("node_ids=%r, path=%r, timeout=%r", node_ids, path, timeout) + node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] + assert isinstance(node_ids_list, list) and all(isinstance(x, int) for x in node_ids_list) + + error = False + with purser.get_node("file_client_remove", allow_anonymous=False) as node: + for nid in node_ids_list: + cln = node.make_client(Modify_1_1, nid) + try: + cln.response_timeout = timeout + resp = await cln(Modify_1_1.Request(source=Path_2_0(path=path))) + if resp is None: + show_error(f"Request to node {nid} has timed out") + error = True + break + assert isinstance(resp, Modify_1_1.Response) + assert isinstance(resp.error, Error_1_0) + if resp.error.value == Error_1_0.OK: + _logger.info("Removed path %r on node %r", path, nid) + elif resp.error.value == Error_1_0.NOT_FOUND: + show_warning(f"Path {path} not found on node {nid}") + else: + show_error(f"Error {FileError(resp.error.value)} while removing {path} on node {nid}") + error = True + + finally: + cln.close() + + return EXIT_CODE_UNSUCCESSFUL if error else 0 + +UNSTRUCTURED_MAX_SIZE = 256 + +@file_client.command() +@click.argument("node_id", type=int) +@click.argument("src") +@click.argument("dst", required=False) +@click.option( + "--timeout", + "-T", + type=float, + default=pycyphal.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, + show_default=True, + metavar="SECONDS", + help="Service response timeout.", +) +@yakut.pass_purser +@yakut.asynchronous(interrupted_ok=True) +async def file_read( + purser: yakut.Purser, + node_id: int, + src: str, + dst: str | None, + timeout: float, +) -> None: + try: + from uavcan.file import Path_2_0 + from uavcan.file import Error_1_0 + from uavcan.file import Read_1_1 + except ImportError as ex: + from yakut.cmd.compile import make_usage_suggestion + + raise click.ClickException(make_usage_suggestion(ex.name)) from None + + src = PurePosixPath(src) + dst = Path(dst) if dst else Path(src.name) + out = None + total_read = 0 + error = False + with purser.get_node("file_client_read", allow_anonymous=False) as node: + prog = ProgressReporter() + cln = node.make_client(Read_1_1, node_id) + cln.response_timeout = timeout + try: + while True: + resp = await cln(Read_1_1.Request(path=Path_2_0(path=str(src)), offset=total_read)) + if resp is None: + show_error(f"Request to node {node_id} has timed out") + error = True + break + assert isinstance(resp, Read_1_1.Response) + assert isinstance(resp.error, Error_1_0) + if resp.error.value != Error_1_0.OK: + show_error(f"Error {FileError(resp.error.value)} while reading {src} on node {node_id}") + error = True + break + + bytes_read = len(resp.data.value) + total_read += bytes_read + prog(f"read {total_read} bytes") + if out is None: + out = open(dst, "wb") + + out.write(resp.data.value) + + if bytes_read < UNSTRUCTURED_MAX_SIZE: + break + finally: + cln.close() + + if out is not None: + out.close() + + return EXIT_CODE_UNSUCCESSFUL if error else 0 + +@file_client.command() +@click.argument("node_id", type=int) +@click.argument("src") +@click.argument("dst", required=False) +@click.option( + "--timeout", + "-T", + type=float, + default=pycyphal.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, + show_default=True, + metavar="SECONDS", + help="Service response timeout.", +) +@yakut.pass_purser +@yakut.asynchronous(interrupted_ok=True) +async def file_write( + purser: yakut.Purser, + node_id: int, + src: str, + dst: str | None, + timeout: float, +) -> None: + try: + from uavcan.file import Path_2_0 + from uavcan.file import Error_1_0 + from uavcan.file import Write_1_1 + from uavcan.primitive import Unstructured_1_0 + except ImportError as ex: + from yakut.cmd.compile import make_usage_suggestion + + raise click.ClickException(make_usage_suggestion(ex.name)) from None + + src = Path(src) + dst = PurePosixPath(dst) if dst else PurePosixPath(src.name) + file = open(src, "rb") + + total_written = 0 + error = False + with purser.get_node("file_client_write", allow_anonymous=False) as node: + prog = ProgressReporter() + cln = node.make_client(Write_1_1, node_id) + cln.response_timeout = timeout + try: + while True: + chunk = file.read(UNSTRUCTURED_MAX_SIZE) + req = Write_1_1.Request( + path=Path_2_0(path=str(dst)), + offset=total_written, + data=Unstructured_1_0(value=chunk) + ) + resp = await cln(req) + if resp is None: + show_error(f"Request to node {node_id} has timed out") + error = True + break + assert isinstance(resp, Write_1_1.Response) + assert isinstance(resp.error, Error_1_0) + if resp.error.value != Error_1_0.OK: + show_error(f"Error {FileError(resp.error.value)} while writing {dst} on node {node_id}") + error = True + break + + bytes_written = len(chunk) + total_written += bytes_written + prog(f"written {total_written} bytes") + + if bytes_written < UNSTRUCTURED_MAX_SIZE: + break + finally: + cln.close() + + file.close() + + return EXIT_CODE_UNSUCCESSFUL if error else 0 diff --git a/yakut/cmd/file_client/_file_error.py b/yakut/cmd/file_client/_file_error.py new file mode 100644 index 0000000..38454f5 --- /dev/null +++ b/yakut/cmd/file_client/_file_error.py @@ -0,0 +1,14 @@ +import enum + +class FileError(enum.IntEnum): + OK = 0 + UNKNWOWN_ERROR = 65535 + + NOT_FOUND = 2 + IO_ERROR = 5 + ACCESS_DENIED = 13 + IS_DIRECTORY = 21 + INVALID_VALUE = 22 + FILE_TOO_LARGE = 27 + OUT_OF_SPACE = 28 + NOT_SUPPORTED = 38 diff --git a/yakut/cmd/file_client/_list_files.py b/yakut/cmd/file_client/_list_files.py new file mode 100644 index 0000000..bb05b7e --- /dev/null +++ b/yakut/cmd/file_client/_list_files.py @@ -0,0 +1,181 @@ +from __future__ import annotations +import dataclasses +import enum +from typing import Sequence, TYPE_CHECKING, Callable +import bisect +import yakut +from ruamel.yaml import YAML +from ._file_error import FileError + +yaml = YAML() + +if TYPE_CHECKING: + import pycyphal.application + +@yaml.register_class +@dataclasses.dataclass +class FileInfo: + size: int + timestamp: int + is_file_not_directory: bool + is_link: bool + is_readable: bool + is_writable: bool + +@yaml.register_class +@dataclasses.dataclass +class FileResult: + name: str + info: FileInfo | FileError | None + +@dataclasses.dataclass +class Result: + files_per_node: dict[int, list[FileResult] | None] = dataclasses.field(default_factory=dict) + errors: list[str] = dataclasses.field(default_factory=list) + warnings: list[str] = dataclasses.field(default_factory=list) + + +async def list_files( + local_node: "pycyphal.application.Node", + progress: Callable[[str], None], + node_ids: Sequence[int], + path: str, + *, + optional_service: bool, + get_info: bool, + timeout: float, +) -> Result: + res = Result() + for nid, names in (await _impl_list_files(local_node, progress, node_ids, path, timeout=timeout)).items(): + _logger.debug("File names @%r: %r", nid, names) + if isinstance(names, _NoService): + res.files_per_node[nid] = None + if optional_service: + res.warnings.append(f"File list service is not accessible at node {nid}, ignoring as requested") + else: + res.errors.append(f"File list service is not accessible at node {nid}") + else: + lst = res.files_per_node.setdefault(nid, []) + assert isinstance(lst, list) + for idx, n in enumerate(names): + if isinstance(n, _Timeout): + res.errors.append(f"Request #{idx} to node {nid} has timed out, data incomplete") + else: + lst.append(FileResult(name=n, info=None)) + if get_info: + finfo = await _impl_get_info( + local_node, + progress, + nid, + path, + [f.name for f in lst], + timeout=timeout, + ) + if isinstance(finfo, _NoService): + if optional_service: + res.warnings.append(f"File info service is not accessible at node {nid}, ignoring as requested") + else: + res.errors.append(f"File info service is not accessible at node {nid}") + else: + for file, info in zip(lst, finfo): + if isinstance(info, _Timeout): + res.errors.append(f"GetInfo for file #{file.name} to node {nid} has timed out, data incomplete") + elif isinstance(info, FileError): + res.errors.append(f"GetInfo error {info} for file {file.name} at node {nid}") + else: + file.info = info + return res + + +class _NoService: + pass + + +class _Timeout: + pass + + +async def _impl_list_files( + local_node: "pycyphal.application.Node", + progress: Callable[[str], None], + node_ids: Sequence[int], + path: str, + *, + timeout: float, +) -> dict[int, list[str | _Timeout] | _NoService]: + from uavcan.file import List_0_2 + from uavcan.file import Path_2_0 + + out: dict[int, list[str | _Timeout] | _NoService] = {} + for nid in node_ids: + cln = local_node.make_client(List_0_2, nid) + try: + cln.response_timeout = timeout + name_list: list[str | _Timeout] | _NoService = [] + for idx in range(2**16): + progress(f"List {nid: 5}: {idx: 5}") + resp = await cln(List_0_2.Request(entry_index=idx, directory_path=Path_2_0(path=path))) + assert isinstance(name_list, list) + if resp is None: + if 0 == idx: # First request timed out, assume service not supported or node is offline + name_list = _NoService() + else: # Non-first request has timed out, assume network error + name_list.append(_Timeout()) + break + assert isinstance(resp, List_0_2.Response) + name = resp.entry_base_name.path.tobytes().decode(errors="replace") + if not name: + break + name_list.append(name) + finally: + cln.close() + _logger.debug("File names fetched from node %r: %r", nid, name_list) + out[nid] = name_list + return out + +async def _impl_get_info( + local_node: "pycyphal.application.Node", + progress: Callable[[str], None], + node_id: int, + path: str, + files: Sequence[str], + *, + timeout: float, +) -> list[FileInfo | _Timeout | None] | _NoService: + from uavcan.file import GetInfo_0_2 + from uavcan.file import Path_2_0 + from uavcan.file import Error_1_0 + + out: list[FileInfo | _Timeout | None] | _NoService = [] + cln = local_node.make_client(GetInfo_0_2, node_id) + for idx, file in enumerate(files): + cln.response_timeout = timeout + progress(f"GetInfo {node_id:5}: {file:5}") + if path != "": + filepath = chr(Path_2_0.SEPARATOR).join([path, file]) + else: + filepath = file + resp = await cln(GetInfo_0_2.Request(path=Path_2_0(path=filepath))) + if resp is None: + if 0 == idx: # First request timed out, assume service not supported or node is offline + out = _NoService() + else: # Non-first request has timed out, assume network error + out.append(_Timeout()) + break + assert isinstance(resp, GetInfo_0_2.Response) + if resp.error.value != Error_1_0.OK: + out.append(FileError(resp.error.value)) + continue + out.append(FileInfo( + size = resp.size, + timestamp=resp.unix_timestamp_of_last_modification, + is_file_not_directory=resp.is_file_not_directory, + is_link=resp.is_link, + is_readable=resp.is_readable, + is_writable=resp.is_writeable + )) + cln.close() + _logger.debug("File info fetched from node %r: %r", node_id, out) + return out + +_logger = yakut.get_logger(__name__) diff --git a/yakut/main.py b/yakut/main.py index 50e0b48..339e29b 100644 --- a/yakut/main.py +++ b/yakut/main.py @@ -290,6 +290,7 @@ def main() -> None: # https://click.palletsprojects.com/en/8.1.x/exceptions/ subcommand: Callable[..., Callable[..., Any]] = _click_main.command # type: ignore +commandgroup: Callable[..., Callable[..., Any]] = _click_main.group # type: ignore def asynchronous(*, interrupted_ok: bool = False) -> Callable[[Callable[..., Awaitable[Any]]], Callable[..., Any]]: From 645c795785e97301e9bb0d181bd990c59af139ca Mon Sep 17 00:00:00 2001 From: chemicstry Date: Wed, 30 Apr 2025 16:52:25 +0300 Subject: [PATCH 2/7] Make command names shorter --- yakut/cmd/file_client/_cmd.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/yakut/cmd/file_client/_cmd.py b/yakut/cmd/file_client/_cmd.py index 122d772..d038225 100644 --- a/yakut/cmd/file_client/_cmd.py +++ b/yakut/cmd/file_client/_cmd.py @@ -17,8 +17,12 @@ from ._file_error import FileError if TYPE_CHECKING: - import pycyphal.application # pylint: disable=ungrouped-imports - import pycyphal.application.file # pylint: disable=ungrouped-imports + from uavcan.file import Path_2_0 + from uavcan.file import Error_1_0 + from uavcan.file import Modify_1_1 + from uavcan.file import Read_1_1 + from uavcan.file import Write_1_1 + from uavcan.primitive import Unstructured_1_0 _logger = yakut.get_logger(__name__) @@ -26,7 +30,7 @@ @click.pass_context @yakut.pass_purser def file_client(purser: yakut.Purser, cmd: str): - """Main CLI group.""" + """File client commands.""" pass @file_client.command() @@ -60,7 +64,7 @@ def file_client(purser: yakut.Purser, cmd: str): ) @yakut.pass_purser @yakut.asynchronous(interrupted_ok=True) -async def file_list( +async def ls( purser: yakut.Purser, node_ids: set[int] | int, path: Path, @@ -72,7 +76,7 @@ async def file_list( node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] assert isinstance(node_ids_list, list) and all(isinstance(x, int) for x in node_ids_list) formatter = purser.make_formatter(FormatterHints(single_document=True)) - with purser.get_node("file_client_list", allow_anonymous=False) as node: + with purser.get_node("file_client_ls", allow_anonymous=False) as node: with ProgressReporter() as prog: result = await list_files( node, @@ -108,7 +112,7 @@ async def file_list( ) @yakut.pass_purser @yakut.asynchronous(interrupted_ok=True) -async def file_remove( +async def rm( purser: yakut.Purser, node_ids: set[int] | int, path: Path, @@ -128,7 +132,7 @@ async def file_remove( assert isinstance(node_ids_list, list) and all(isinstance(x, int) for x in node_ids_list) error = False - with purser.get_node("file_client_remove", allow_anonymous=False) as node: + with purser.get_node("file_client_rm", allow_anonymous=False) as node: for nid in node_ids_list: cln = node.make_client(Modify_1_1, nid) try: @@ -170,7 +174,7 @@ async def file_remove( ) @yakut.pass_purser @yakut.asynchronous(interrupted_ok=True) -async def file_read( +async def read( purser: yakut.Purser, node_id: int, src: str, @@ -242,7 +246,7 @@ async def file_read( ) @yakut.pass_purser @yakut.asynchronous(interrupted_ok=True) -async def file_write( +async def write( purser: yakut.Purser, node_id: int, src: str, From 1ebb3b81c9ef28779095b6337508c8a66901c060 Mon Sep 17 00:00:00 2001 From: chemicstry Date: Mon, 19 May 2025 16:30:50 +0300 Subject: [PATCH 3/7] Rewrite on FileClient2 --- yakut/cmd/file_client/__init__.py | 1 - yakut/cmd/file_client/_cmd.py | 253 ++++++++++++++------------- yakut/cmd/file_client/_file_error.py | 14 -- yakut/cmd/file_client/_list_files.py | 181 ------------------- 4 files changed, 131 insertions(+), 318 deletions(-) delete mode 100644 yakut/cmd/file_client/_file_error.py delete mode 100644 yakut/cmd/file_client/_list_files.py diff --git a/yakut/cmd/file_client/__init__.py b/yakut/cmd/file_client/__init__.py index b0ce018..3e16a60 100644 --- a/yakut/cmd/file_client/__init__.py +++ b/yakut/cmd/file_client/__init__.py @@ -2,5 +2,4 @@ # This software is distributed under the terms of the MIT License. # Author: Pavel Kirienko -import enum from ._cmd import file_client as file_client diff --git a/yakut/cmd/file_client/_cmd.py b/yakut/cmd/file_client/_cmd.py index d038225..ea20c62 100644 --- a/yakut/cmd/file_client/_cmd.py +++ b/yakut/cmd/file_client/_cmd.py @@ -4,7 +4,6 @@ from __future__ import annotations import sys -from typing import TYPE_CHECKING from pathlib import Path, PurePosixPath import click import pycyphal @@ -13,16 +12,26 @@ from yakut.param.formatter import FormatterHints from yakut.ui import ProgressReporter, show_error, show_warning from yakut.util import EXIT_CODE_UNSUCCESSFUL -from ._list_files import list_files -from ._file_error import FileError +from ruamel.yaml import YAML +import dataclasses -if TYPE_CHECKING: - from uavcan.file import Path_2_0 - from uavcan.file import Error_1_0 - from uavcan.file import Modify_1_1 - from uavcan.file import Read_1_1 - from uavcan.file import Write_1_1 - from uavcan.primitive import Unstructured_1_0 +yaml = YAML() + +@yaml.register_class +@dataclasses.dataclass +class FileInfo: + size: int + timestamp: int + is_file_not_directory: bool + is_link: bool + is_readable: bool + is_writable: bool + +@yaml.register_class +@dataclasses.dataclass +class FileResult: + name: str + info: FileInfo | None _logger = yakut.get_logger(__name__) @@ -53,7 +62,7 @@ def file_client(purser: yakut.Purser, cmd: str): Ignore nodes that fail to respond to the first RPC-service request instead of reporting an error assuming that the register service is not supported. If a node responded at least once it is assumed to support the service and any future timeout -will be always treated as an error. +will be treated as an error. """, ) @click.option( @@ -72,31 +81,68 @@ async def ls( optional_service: bool, get_info: bool, ) -> None: + """ + List files on a remote node using the standard Cyphal file service. + """ + try: + from pycyphal.application.file import FileClient2 + from uavcan.file import Path_2_0 + except ImportError as ex: + from yakut.cmd.compile import make_usage_suggestion + raise click.ClickException(make_usage_suggestion(ex.name)) + _logger.debug("node_ids=%r, path=%r, timeout=%r", node_ids, path, timeout) node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] assert isinstance(node_ids_list, list) and all(isinstance(x, int) for x in node_ids_list) + + errors: list[str] = [] + warnings: list[str] = [] + files_per_node: dict[int, list[str]] = {} + formatter = purser.make_formatter(FormatterHints(single_document=True)) + with purser.get_node("file_client_ls", allow_anonymous=False) as node: - with ProgressReporter() as prog: - result = await list_files( - node, - prog, - node_ids_list, - path, - optional_service=optional_service, - get_info=get_info, - timeout=timeout, - ) - # The node is no longer needed. - for msg in result.errors: + prog = ProgressReporter() + for nid in node_ids_list: + files = [] + try: + fc = FileClient2(node, nid, response_timeout=timeout) + async for entry in fc.list(str(path)): + prog(f"List {nid: 5}: {len(files): 5}") + info = None + if get_info: + try: + filepath = chr(Path_2_0.SEPARATOR).join([path, entry]) + resp = await fc.get_info(filepath) + info = FileInfo( + size = resp.size, + timestamp=resp.unix_timestamp_of_last_modification, + is_file_not_directory=resp.is_file_not_directory, + is_link=resp.is_link, + is_readable=resp.is_readable, + is_writable=resp.is_writeable + ) + except Exception as e: + warnings.append(f"Could not get info for {path}/{entry} from node {nid}: {e}") + + files.append(dataclasses.asdict(FileResult(name=entry, info=info))) + + files_per_node[nid] = files + + except Exception as e: + if not (optional_service and "not supported" in str(e).lower()): + errors.append(f"Error listing {path} from node {nid}: {e}") + + for msg in errors: show_error(msg) - for msg in result.warnings: + for msg in warnings: show_warning(msg) - final = result.files_per_node if not isinstance(node_ids, int) else result.files_per_node[node_ids] + + final = files_per_node if not isinstance(node_ids, int) else files_per_node[node_ids] sys.stdout.write(formatter(final)) sys.stdout.flush() - return EXIT_CODE_UNSUCCESSFUL if result.errors else 0 + return yakut.util.EXIT_CODE_UNSUCCESSFUL if errors else 0 @file_client.command() @click.argument("node_ids", type=parse_int_set) @@ -118,14 +164,14 @@ async def rm( path: Path, timeout: float, ) -> None: + """ + Remove a file or directory on remote node(s) using the standard Cyphal file service. + """ try: - from uavcan.file import Path_2_0 - from uavcan.file import Error_1_0 - from uavcan.file import Modify_1_1 + from pycyphal.application.file import FileClient2 except ImportError as ex: from yakut.cmd.compile import make_usage_suggestion - - raise click.ClickException(make_usage_suggestion(ex.name)) from None + raise click.ClickException(make_usage_suggestion(ex.name)) _logger.debug("node_ids=%r, path=%r, timeout=%r", node_ids, path, timeout) node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] @@ -133,27 +179,24 @@ async def rm( error = False with purser.get_node("file_client_rm", allow_anonymous=False) as node: + prog = ProgressReporter() for nid in node_ids_list: - cln = node.make_client(Modify_1_1, nid) try: - cln.response_timeout = timeout - resp = await cln(Modify_1_1.Request(source=Path_2_0(path=path))) - if resp is None: - show_error(f"Request to node {nid} has timed out") - error = True - break - assert isinstance(resp, Modify_1_1.Response) - assert isinstance(resp.error, Error_1_0) - if resp.error.value == Error_1_0.OK: + fc = FileClient2(node, nid, response_timeout=timeout) + prog(f"Remove from node {nid}") + + try: + await fc.remove(str(path)) _logger.info("Removed path %r on node %r", path, nid) - elif resp.error.value == Error_1_0.NOT_FOUND: + except FileNotFoundError: show_warning(f"Path {path} not found on node {nid}") - else: - show_error(f"Error {FileError(resp.error.value)} while removing {path} on node {nid}") + except Exception as e: + show_error(f"Error removing {path} on node {nid}: {e}") error = True - - finally: - cln.close() + + except Exception as e: + show_error(f"Error setting up file client for node {nid}: {e}") + error = True return EXIT_CODE_UNSUCCESSFUL if error else 0 @@ -181,53 +224,39 @@ async def read( dst: str | None, timeout: float, ) -> None: + """ + Read a file from a remote node using the standard Cyphal file service. + """ try: - from uavcan.file import Path_2_0 - from uavcan.file import Error_1_0 - from uavcan.file import Read_1_1 + from pycyphal.application.file import FileClient2 except ImportError as ex: from yakut.cmd.compile import make_usage_suggestion - - raise click.ClickException(make_usage_suggestion(ex.name)) from None + raise click.ClickException(make_usage_suggestion(ex.name)) src = PurePosixPath(src) dst = Path(dst) if dst else Path(src.name) - out = None - total_read = 0 error = False + with purser.get_node("file_client_read", allow_anonymous=False) as node: prog = ProgressReporter() - cln = node.make_client(Read_1_1, node_id) - cln.response_timeout = timeout - try: - while True: - resp = await cln(Read_1_1.Request(path=Path_2_0(path=str(src)), offset=total_read)) - if resp is None: - show_error(f"Request to node {node_id} has timed out") - error = True - break - assert isinstance(resp, Read_1_1.Response) - assert isinstance(resp.error, Error_1_0) - if resp.error.value != Error_1_0.OK: - show_error(f"Error {FileError(resp.error.value)} while reading {src} on node {node_id}") - error = True - break + def read_progress_cb(bytes_read: int, bytes_total: int | None) -> None: + prog(f"Read {bytes_read} bytes") - bytes_read = len(resp.data.value) - total_read += bytes_read - prog(f"read {total_read} bytes") - if out is None: - out = open(dst, "wb") + try: + fc = FileClient2(node, node_id, response_timeout=timeout) + + with open(dst, "wb") as out: + res = await fc.read(str(src), progress=read_progress_cb) + out.write(res) - out.write(resp.data.value) + _logger.info("Read %d bytes from %r on node %r to %r", len(res), src, node_id, dst) - if bytes_read < UNSTRUCTURED_MAX_SIZE: - break - finally: - cln.close() - - if out is not None: - out.close() + except FileNotFoundError: + show_error(f"File {src} not found on node {node_id}") + error = True + except Exception as e: + show_error(f"Error reading {src} from node {node_id}: {e}") + error = True return EXIT_CODE_UNSUCCESSFUL if error else 0 @@ -253,55 +282,35 @@ async def write( dst: str | None, timeout: float, ) -> None: + """ + Write a file to a remote node using the standard Cyphal file service. + """ try: - from uavcan.file import Path_2_0 - from uavcan.file import Error_1_0 - from uavcan.file import Write_1_1 - from uavcan.primitive import Unstructured_1_0 + from pycyphal.application.file import FileClient2 except ImportError as ex: from yakut.cmd.compile import make_usage_suggestion - - raise click.ClickException(make_usage_suggestion(ex.name)) from None + raise click.ClickException(make_usage_suggestion(ex.name)) src = Path(src) dst = PurePosixPath(dst) if dst else PurePosixPath(src.name) - file = open(src, "rb") - - total_written = 0 error = False + with purser.get_node("file_client_write", allow_anonymous=False) as node: prog = ProgressReporter() - cln = node.make_client(Write_1_1, node_id) - cln.response_timeout = timeout + def write_progress_cb(bytes_written: int, bytes_total: int | None) -> None: + prog(f"Written {bytes_written}/{bytes_total} bytes") + try: - while True: - chunk = file.read(UNSTRUCTURED_MAX_SIZE) - req = Write_1_1.Request( - path=Path_2_0(path=str(dst)), - offset=total_written, - data=Unstructured_1_0(value=chunk) - ) - resp = await cln(req) - if resp is None: - show_error(f"Request to node {node_id} has timed out") - error = True - break - assert isinstance(resp, Write_1_1.Response) - assert isinstance(resp.error, Error_1_0) - if resp.error.value != Error_1_0.OK: - show_error(f"Error {FileError(resp.error.value)} while writing {dst} on node {node_id}") - error = True - break - - bytes_written = len(chunk) - total_written += bytes_written - prog(f"written {total_written} bytes") - - if bytes_written < UNSTRUCTURED_MAX_SIZE: - break - finally: - cln.close() - - file.close() + fc = FileClient2(node, node_id, response_timeout=timeout) + + with open(src, "rb") as file: + data = file.read() + await fc.write(str(dst), data, progress=write_progress_cb) + + _logger.info("Written %d bytes from %r to %r on node %r", len(data), src, dst, node_id) + + except Exception as e: + show_error(f"Error writing {src} to node {node_id}: {e}") + error = True return EXIT_CODE_UNSUCCESSFUL if error else 0 diff --git a/yakut/cmd/file_client/_file_error.py b/yakut/cmd/file_client/_file_error.py deleted file mode 100644 index 38454f5..0000000 --- a/yakut/cmd/file_client/_file_error.py +++ /dev/null @@ -1,14 +0,0 @@ -import enum - -class FileError(enum.IntEnum): - OK = 0 - UNKNWOWN_ERROR = 65535 - - NOT_FOUND = 2 - IO_ERROR = 5 - ACCESS_DENIED = 13 - IS_DIRECTORY = 21 - INVALID_VALUE = 22 - FILE_TOO_LARGE = 27 - OUT_OF_SPACE = 28 - NOT_SUPPORTED = 38 diff --git a/yakut/cmd/file_client/_list_files.py b/yakut/cmd/file_client/_list_files.py deleted file mode 100644 index bb05b7e..0000000 --- a/yakut/cmd/file_client/_list_files.py +++ /dev/null @@ -1,181 +0,0 @@ -from __future__ import annotations -import dataclasses -import enum -from typing import Sequence, TYPE_CHECKING, Callable -import bisect -import yakut -from ruamel.yaml import YAML -from ._file_error import FileError - -yaml = YAML() - -if TYPE_CHECKING: - import pycyphal.application - -@yaml.register_class -@dataclasses.dataclass -class FileInfo: - size: int - timestamp: int - is_file_not_directory: bool - is_link: bool - is_readable: bool - is_writable: bool - -@yaml.register_class -@dataclasses.dataclass -class FileResult: - name: str - info: FileInfo | FileError | None - -@dataclasses.dataclass -class Result: - files_per_node: dict[int, list[FileResult] | None] = dataclasses.field(default_factory=dict) - errors: list[str] = dataclasses.field(default_factory=list) - warnings: list[str] = dataclasses.field(default_factory=list) - - -async def list_files( - local_node: "pycyphal.application.Node", - progress: Callable[[str], None], - node_ids: Sequence[int], - path: str, - *, - optional_service: bool, - get_info: bool, - timeout: float, -) -> Result: - res = Result() - for nid, names in (await _impl_list_files(local_node, progress, node_ids, path, timeout=timeout)).items(): - _logger.debug("File names @%r: %r", nid, names) - if isinstance(names, _NoService): - res.files_per_node[nid] = None - if optional_service: - res.warnings.append(f"File list service is not accessible at node {nid}, ignoring as requested") - else: - res.errors.append(f"File list service is not accessible at node {nid}") - else: - lst = res.files_per_node.setdefault(nid, []) - assert isinstance(lst, list) - for idx, n in enumerate(names): - if isinstance(n, _Timeout): - res.errors.append(f"Request #{idx} to node {nid} has timed out, data incomplete") - else: - lst.append(FileResult(name=n, info=None)) - if get_info: - finfo = await _impl_get_info( - local_node, - progress, - nid, - path, - [f.name for f in lst], - timeout=timeout, - ) - if isinstance(finfo, _NoService): - if optional_service: - res.warnings.append(f"File info service is not accessible at node {nid}, ignoring as requested") - else: - res.errors.append(f"File info service is not accessible at node {nid}") - else: - for file, info in zip(lst, finfo): - if isinstance(info, _Timeout): - res.errors.append(f"GetInfo for file #{file.name} to node {nid} has timed out, data incomplete") - elif isinstance(info, FileError): - res.errors.append(f"GetInfo error {info} for file {file.name} at node {nid}") - else: - file.info = info - return res - - -class _NoService: - pass - - -class _Timeout: - pass - - -async def _impl_list_files( - local_node: "pycyphal.application.Node", - progress: Callable[[str], None], - node_ids: Sequence[int], - path: str, - *, - timeout: float, -) -> dict[int, list[str | _Timeout] | _NoService]: - from uavcan.file import List_0_2 - from uavcan.file import Path_2_0 - - out: dict[int, list[str | _Timeout] | _NoService] = {} - for nid in node_ids: - cln = local_node.make_client(List_0_2, nid) - try: - cln.response_timeout = timeout - name_list: list[str | _Timeout] | _NoService = [] - for idx in range(2**16): - progress(f"List {nid: 5}: {idx: 5}") - resp = await cln(List_0_2.Request(entry_index=idx, directory_path=Path_2_0(path=path))) - assert isinstance(name_list, list) - if resp is None: - if 0 == idx: # First request timed out, assume service not supported or node is offline - name_list = _NoService() - else: # Non-first request has timed out, assume network error - name_list.append(_Timeout()) - break - assert isinstance(resp, List_0_2.Response) - name = resp.entry_base_name.path.tobytes().decode(errors="replace") - if not name: - break - name_list.append(name) - finally: - cln.close() - _logger.debug("File names fetched from node %r: %r", nid, name_list) - out[nid] = name_list - return out - -async def _impl_get_info( - local_node: "pycyphal.application.Node", - progress: Callable[[str], None], - node_id: int, - path: str, - files: Sequence[str], - *, - timeout: float, -) -> list[FileInfo | _Timeout | None] | _NoService: - from uavcan.file import GetInfo_0_2 - from uavcan.file import Path_2_0 - from uavcan.file import Error_1_0 - - out: list[FileInfo | _Timeout | None] | _NoService = [] - cln = local_node.make_client(GetInfo_0_2, node_id) - for idx, file in enumerate(files): - cln.response_timeout = timeout - progress(f"GetInfo {node_id:5}: {file:5}") - if path != "": - filepath = chr(Path_2_0.SEPARATOR).join([path, file]) - else: - filepath = file - resp = await cln(GetInfo_0_2.Request(path=Path_2_0(path=filepath))) - if resp is None: - if 0 == idx: # First request timed out, assume service not supported or node is offline - out = _NoService() - else: # Non-first request has timed out, assume network error - out.append(_Timeout()) - break - assert isinstance(resp, GetInfo_0_2.Response) - if resp.error.value != Error_1_0.OK: - out.append(FileError(resp.error.value)) - continue - out.append(FileInfo( - size = resp.size, - timestamp=resp.unix_timestamp_of_last_modification, - is_file_not_directory=resp.is_file_not_directory, - is_link=resp.is_link, - is_readable=resp.is_readable, - is_writable=resp.is_writeable - )) - cln.close() - _logger.debug("File info fetched from node %r: %r", node_id, out) - return out - -_logger = yakut.get_logger(__name__) From 5c24275a0402a46ae9c2bf8c7378af2e07921219 Mon Sep 17 00:00:00 2001 From: chemicstry Date: Mon, 19 May 2025 16:37:29 +0300 Subject: [PATCH 4/7] Formatting --- yakut/cmd/file_client/_cmd.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/yakut/cmd/file_client/_cmd.py b/yakut/cmd/file_client/_cmd.py index ea20c62..3ed67a5 100644 --- a/yakut/cmd/file_client/_cmd.py +++ b/yakut/cmd/file_client/_cmd.py @@ -12,12 +12,8 @@ from yakut.param.formatter import FormatterHints from yakut.ui import ProgressReporter, show_error, show_warning from yakut.util import EXIT_CODE_UNSUCCESSFUL -from ruamel.yaml import YAML import dataclasses -yaml = YAML() - -@yaml.register_class @dataclasses.dataclass class FileInfo: size: int @@ -27,14 +23,16 @@ class FileInfo: is_readable: bool is_writable: bool -@yaml.register_class + @dataclasses.dataclass class FileResult: name: str info: FileInfo | None + _logger = yakut.get_logger(__name__) + @yakut.commandgroup(aliases="fcli") @click.pass_context @yakut.pass_purser @@ -42,6 +40,7 @@ def file_client(purser: yakut.Purser, cmd: str): """File client commands.""" pass + @file_client.command() @click.argument("node_ids", type=parse_int_set) @click.argument("path", default="") @@ -89,6 +88,7 @@ async def ls( from uavcan.file import Path_2_0 except ImportError as ex: from yakut.cmd.compile import make_usage_suggestion + raise click.ClickException(make_usage_suggestion(ex.name)) _logger.debug("node_ids=%r, path=%r, timeout=%r", node_ids, path, timeout) @@ -115,12 +115,12 @@ async def ls( filepath = chr(Path_2_0.SEPARATOR).join([path, entry]) resp = await fc.get_info(filepath) info = FileInfo( - size = resp.size, + size=resp.size, timestamp=resp.unix_timestamp_of_last_modification, is_file_not_directory=resp.is_file_not_directory, is_link=resp.is_link, is_readable=resp.is_readable, - is_writable=resp.is_writeable + is_writable=resp.is_writeable, ) except Exception as e: warnings.append(f"Could not get info for {path}/{entry} from node {nid}: {e}") @@ -144,6 +144,7 @@ async def ls( return yakut.util.EXIT_CODE_UNSUCCESSFUL if errors else 0 + @file_client.command() @click.argument("node_ids", type=parse_int_set) @click.argument("path", default="") @@ -171,6 +172,7 @@ async def rm( from pycyphal.application.file import FileClient2 except ImportError as ex: from yakut.cmd.compile import make_usage_suggestion + raise click.ClickException(make_usage_suggestion(ex.name)) _logger.debug("node_ids=%r, path=%r, timeout=%r", node_ids, path, timeout) @@ -184,7 +186,7 @@ async def rm( try: fc = FileClient2(node, nid, response_timeout=timeout) prog(f"Remove from node {nid}") - + try: await fc.remove(str(path)) _logger.info("Removed path %r on node %r", path, nid) @@ -200,8 +202,10 @@ async def rm( return EXIT_CODE_UNSUCCESSFUL if error else 0 + UNSTRUCTURED_MAX_SIZE = 256 + @file_client.command() @click.argument("node_id", type=int) @click.argument("src") @@ -231,6 +235,7 @@ async def read( from pycyphal.application.file import FileClient2 except ImportError as ex: from yakut.cmd.compile import make_usage_suggestion + raise click.ClickException(make_usage_suggestion(ex.name)) src = PurePosixPath(src) @@ -239,16 +244,17 @@ async def read( with purser.get_node("file_client_read", allow_anonymous=False) as node: prog = ProgressReporter() + def read_progress_cb(bytes_read: int, bytes_total: int | None) -> None: prog(f"Read {bytes_read} bytes") try: fc = FileClient2(node, node_id, response_timeout=timeout) - + with open(dst, "wb") as out: res = await fc.read(str(src), progress=read_progress_cb) out.write(res) - + _logger.info("Read %d bytes from %r on node %r to %r", len(res), src, node_id, dst) except FileNotFoundError: @@ -260,6 +266,7 @@ def read_progress_cb(bytes_read: int, bytes_total: int | None) -> None: return EXIT_CODE_UNSUCCESSFUL if error else 0 + @file_client.command() @click.argument("node_id", type=int) @click.argument("src") @@ -289,6 +296,7 @@ async def write( from pycyphal.application.file import FileClient2 except ImportError as ex: from yakut.cmd.compile import make_usage_suggestion + raise click.ClickException(make_usage_suggestion(ex.name)) src = Path(src) @@ -297,16 +305,17 @@ async def write( with purser.get_node("file_client_write", allow_anonymous=False) as node: prog = ProgressReporter() + def write_progress_cb(bytes_written: int, bytes_total: int | None) -> None: prog(f"Written {bytes_written}/{bytes_total} bytes") try: fc = FileClient2(node, node_id, response_timeout=timeout) - + with open(src, "rb") as file: data = file.read() await fc.write(str(dst), data, progress=write_progress_cb) - + _logger.info("Written %d bytes from %r to %r on node %r", len(data), src, dst, node_id) except Exception as e: From cb34afed819d6dce0749e84437e2c001234c489b Mon Sep 17 00:00:00 2001 From: chemicstry Date: Tue, 20 May 2025 13:48:05 +0300 Subject: [PATCH 5/7] Refactor and add tests --- tests/cmd/file_client.py | 164 +++++++++++++++++++++++++++++ yakut/cmd/file_client/_cmd.py | 193 ++++++++++++++++++++++++++-------- 2 files changed, 316 insertions(+), 41 deletions(-) create mode 100644 tests/cmd/file_client.py diff --git a/tests/cmd/file_client.py b/tests/cmd/file_client.py new file mode 100644 index 0000000..36f425c --- /dev/null +++ b/tests/cmd/file_client.py @@ -0,0 +1,164 @@ +# Copyright (c) 2021 OpenCyphal +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +from __future__ import annotations +import asyncio +import tempfile +import shutil +import json +import pytest +from pathlib import Path +from yakut.util import EXIT_CODE_UNSUCCESSFUL +from tests.subprocess import Subprocess + + +async def _setup_test_env(): + server_root = tempfile.mkdtemp(".file_server", "root.") + client_root = tempfile.mkdtemp(".file_client", "root.") + print("SERVER ROOT:", server_root) + print("CLIENT ROOT:", client_root) + + # Start file server in background + srv_proc = Subprocess.cli( + "file-server", + server_root, + environment_variables={"UAVCAN__UDP__IFACE": "127.0.0.1", "UAVCAN__NODE__ID": "42"}, + ) + await asyncio.sleep(5.0) # Let the server initialize + assert srv_proc.alive + return server_root, client_root, srv_proc + + +async def _cleanup_test_env(server_root: str, client_root: str, srv_proc: Subprocess): + srv_proc.wait(10.0, interrupt=True) + await asyncio.sleep(2.0) + shutil.rmtree(server_root, ignore_errors=True) + shutil.rmtree(client_root, ignore_errors=True) + + +async def _run_client_command(command: str, *args: str) -> tuple[int, str, str]: + """Helper to run file client commands with common configuration""" + proc = Subprocess.cli( + "-j", + "file-client", + command, + *args, + environment_variables={ + "UAVCAN__UDP__IFACE": "127.0.0.1", + "UAVCAN__NODE__ID": "43", + }, + ) + return proc.wait(10.0) + + +async def _unittest_file_client_basic_operations() -> None: + """Test basic file operations: ls, touch, read, write, rm""" + server_root, client_root, srv_proc = await _setup_test_env() + try: + # List empty directory + exitcode, stdout, stderr = await _run_client_command("ls", "42", "/") + print(stderr) + assert exitcode == 0 + files = json.loads(stdout) + print(files) + assert isinstance(files, list) + assert len(files) == 0 # Empty directory should show empty list + + # Create a test file + exitcode, stdout, _ = await _run_client_command("touch", "42", "/test.txt") + assert exitcode == 0 + + # Verify file exists with ls + exitcode, stdout, _ = await _run_client_command("ls", "42", "/") + assert exitcode == 0 + files = json.loads(stdout) + assert isinstance(files, list) + assert any(f["name"] == "test.txt" for f in files) + + # Write content to file + test_content = "Hello, World!" + temp_file = Path(client_root) / "local_test.txt" + temp_file.write_text(test_content) + exitcode, _, _ = await _run_client_command("write", "42", str(temp_file), "/test.txt") + assert exitcode == 0 + + # Read back the content + read_file = Path(client_root) / "read_test.txt" + exitcode, _, _ = await _run_client_command("read", "42", "/test.txt", str(read_file)) + assert exitcode == 0 + assert read_file.read_text() == test_content + + # Copy the file + exitcode, _, _ = await _run_client_command("cp", "42", "/test.txt", "/copy.txt") + assert exitcode == 0 + + # Verify both files exist + exitcode, stdout, _ = await _run_client_command("ls", "42", "/") + assert exitcode == 0 + print(stdout) + files = json.loads(stdout) + assert isinstance(files, list) + filenames = [f["name"] for f in files] + assert all(name in filenames for name in ["test.txt", "copy.txt"]) + + # Move the source file + exitcode, _, _ = await _run_client_command("mv", "42", "/test.txt", "/moved.txt") + assert exitcode == 0 + + # Verify file list after move + exitcode, stdout, _ = await _run_client_command("ls", "42", "/") + assert exitcode == 0 + files = json.loads(stdout) + assert isinstance(files, list) + filenames = [f["name"] for f in files] + assert all(name in filenames for name in ["moved.txt", "copy.txt"]) + assert "test.txt" not in filenames + + # Remove the file + exitcode, _, _ = await _run_client_command("rm", "42", "/moved.txt") + assert exitcode == 0 + + # Verify file is gone + exitcode, stdout, _ = await _run_client_command("ls", "42", "/") + assert exitcode == 0 + files = json.loads(stdout) + assert isinstance(files, list) + assert not any(f["name"] == "moved.txt" for f in files) + + finally: + await _cleanup_test_env(server_root, client_root, srv_proc) + + +async def _unittest_file_client_error_cases() -> None: + """Test error handling in file client operations""" + server_root, client_root, srv_proc = await _setup_test_env() + try: + # Try to read non-existent file + exitcode, _, stderr = await _run_client_command("read", "42", "/nonexistent.txt", str(Path(client_root) / "local.txt")) + assert exitcode == EXIT_CODE_UNSUCCESSFUL + assert "not found" in stderr.lower() + + # Try to remove non-existent file (warning) + exitcode, _, stderr = await _run_client_command("rm", "42", "/nonexistent.txt") + assert exitcode == 0 + assert "not found" in stderr.lower() + + # Create a file then try invalid operations + exitcode, _, _ = await _run_client_command("touch", "42", "/test.txt") + assert exitcode == 0 + + # Try to create file that already exists (should work, just updates timestamp) + exitcode, _, _ = await _run_client_command("touch", "42", "/test.txt") + assert exitcode == 0 + + # Try to move to invalid destination (root is read-only) + exitcode, _, stderr = await _run_client_command("mv", "42", "/test.txt", "//test.txt") + assert exitcode == EXIT_CODE_UNSUCCESSFUL + + # Try to write with invalid node ID + exitcode, _, stderr = await _run_client_command("write", "999", "/test.txt", str(Path(client_root) / "local.txt")) + assert exitcode == EXIT_CODE_UNSUCCESSFUL + + finally: + await _cleanup_test_env(server_root, client_root, srv_proc) diff --git a/yakut/cmd/file_client/_cmd.py b/yakut/cmd/file_client/_cmd.py index 3ed67a5..9251e35 100644 --- a/yakut/cmd/file_client/_cmd.py +++ b/yakut/cmd/file_client/_cmd.py @@ -12,6 +12,7 @@ from yakut.param.formatter import FormatterHints from yakut.ui import ProgressReporter, show_error, show_warning from yakut.util import EXIT_CODE_UNSUCCESSFUL +from pycyphal.application.file import FileClient2 import dataclasses @dataclasses.dataclass @@ -83,13 +84,6 @@ async def ls( """ List files on a remote node using the standard Cyphal file service. """ - try: - from pycyphal.application.file import FileClient2 - from uavcan.file import Path_2_0 - except ImportError as ex: - from yakut.cmd.compile import make_usage_suggestion - - raise click.ClickException(make_usage_suggestion(ex.name)) _logger.debug("node_ids=%r, path=%r, timeout=%r", node_ids, path, timeout) node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] @@ -145,6 +139,149 @@ async def ls( return yakut.util.EXIT_CODE_UNSUCCESSFUL if errors else 0 +@file_client.command() +@click.argument("node_ids", type=parse_int_set) +@click.argument("src") +@click.argument("dst") +@click.option( + "--timeout", + "-T", + type=float, + default=pycyphal.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, + show_default=True, + metavar="SECONDS", + help="Service response timeout.", +) +@yakut.pass_purser +@yakut.asynchronous(interrupted_ok=True) +async def mv( + purser: yakut.Purser, + node_ids: set[int] | int, + src: str, + dst: str, + timeout: float, +) -> None: + """ + Move/rename a file or directory on remote node(s) using the standard Cyphal file service. + """ + + _logger.debug("node_ids=%r, src=%r, dst=%r, timeout=%r", node_ids, src, dst, timeout) + node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] + assert isinstance(node_ids_list, list) and all(isinstance(x, int) for x in node_ids_list) + + error = False + with purser.get_node("file_client_mv", allow_anonymous=False) as node: + prog = ProgressReporter() + for nid in node_ids_list: + fc = FileClient2(node, nid, response_timeout=timeout) + prog(f"Moving on node {nid}") + + try: + await fc.move(str(src), str(dst)) + _logger.info("Moved %r to %r on node %r", src, dst, nid) + except FileNotFoundError: + show_warning(f"Source path {src} not found on node {nid}") + except Exception as e: + show_error(f"Error moving {src} to {dst} on node {nid}: {e}") + error = True + + return EXIT_CODE_UNSUCCESSFUL if error else 0 + + +@file_client.command() +@click.argument("node_ids", type=parse_int_set) +@click.argument("src") +@click.argument("dst") +@click.option( + "--timeout", + "-T", + type=float, + default=pycyphal.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, + show_default=True, + metavar="SECONDS", + help="Service response timeout.", +) +@yakut.pass_purser +@yakut.asynchronous(interrupted_ok=True) +async def cp( + purser: yakut.Purser, + node_ids: set[int] | int, + src: str, + dst: str, + timeout: float, +) -> None: + """ + Copy a file on remote node(s) using the standard Cyphal file service. + """ + + _logger.debug("node_ids=%r, src=%r, dst=%r, timeout=%r", node_ids, src, dst, timeout) + node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] + assert isinstance(node_ids_list, list) and all(isinstance(x, int) for x in node_ids_list) + + error = False + with purser.get_node("file_client_cp", allow_anonymous=False) as node: + prog = ProgressReporter() + for nid in node_ids_list: + fc = FileClient2(node, nid, response_timeout=timeout) + prog(f"Copying on node {nid}") + + try: + await fc.copy(str(src), str(dst)) + _logger.info("Copied %r to %r on node %r", src, dst, nid) + except FileNotFoundError: + show_warning(f"Source path {src} not found on node {nid}") + except Exception as e: + show_error(f"Error copying {src} to {dst} on node {nid}: {e}") + error = True + + return EXIT_CODE_UNSUCCESSFUL if error else 0 + + +@file_client.command() +@click.argument("node_ids", type=parse_int_set) +@click.argument("path") +@click.option( + "--timeout", + "-T", + type=float, + default=pycyphal.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, + show_default=True, + metavar="SECONDS", + help="Service response timeout.", +) +@yakut.pass_purser +@yakut.asynchronous(interrupted_ok=True) +async def touch( + purser: yakut.Purser, + node_ids: set[int] | int, + path: str, + timeout: float, +) -> None: + """ + Create an empty file or update timestamp on remote node(s) using the standard Cyphal file service. + """ + + _logger.debug("node_ids=%r, path=%r, timeout=%r", node_ids, path, timeout) + node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] + assert isinstance(node_ids_list, list) and all(isinstance(x, int) for x in node_ids_list) + + error = False + with purser.get_node("file_client_touch", allow_anonymous=False) as node: + prog = ProgressReporter() + for nid in node_ids_list: + fc = FileClient2(node, nid, response_timeout=timeout) + prog(f"Touching on node {nid}") + + try: + await fc.touch(str(path)) + _logger.info("Touched %r on node %r", path, nid) + except Exception as e: + show_error(f"Error touching {path} on node {nid}: {e}") + error = True + + return EXIT_CODE_UNSUCCESSFUL if error else 0 + + @file_client.command() @click.argument("node_ids", type=parse_int_set) @click.argument("path", default="") @@ -168,12 +305,6 @@ async def rm( """ Remove a file or directory on remote node(s) using the standard Cyphal file service. """ - try: - from pycyphal.application.file import FileClient2 - except ImportError as ex: - from yakut.cmd.compile import make_usage_suggestion - - raise click.ClickException(make_usage_suggestion(ex.name)) _logger.debug("node_ids=%r, path=%r, timeout=%r", node_ids, path, timeout) node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] @@ -183,29 +314,21 @@ async def rm( with purser.get_node("file_client_rm", allow_anonymous=False) as node: prog = ProgressReporter() for nid in node_ids_list: - try: - fc = FileClient2(node, nid, response_timeout=timeout) - prog(f"Remove from node {nid}") - - try: - await fc.remove(str(path)) - _logger.info("Removed path %r on node %r", path, nid) - except FileNotFoundError: - show_warning(f"Path {path} not found on node {nid}") - except Exception as e: - show_error(f"Error removing {path} on node {nid}: {e}") - error = True + fc = FileClient2(node, nid, response_timeout=timeout) + prog(f"Remove from node {nid}") + try: + await fc.remove(str(path)) + _logger.info("Removed path %r on node %r", path, nid) + except FileNotFoundError: + show_warning(f"Path {path} not found on node {nid}") except Exception as e: - show_error(f"Error setting up file client for node {nid}: {e}") + show_error(f"Error removing {path} on node {nid}: {e}") error = True return EXIT_CODE_UNSUCCESSFUL if error else 0 -UNSTRUCTURED_MAX_SIZE = 256 - - @file_client.command() @click.argument("node_id", type=int) @click.argument("src") @@ -231,12 +354,6 @@ async def read( """ Read a file from a remote node using the standard Cyphal file service. """ - try: - from pycyphal.application.file import FileClient2 - except ImportError as ex: - from yakut.cmd.compile import make_usage_suggestion - - raise click.ClickException(make_usage_suggestion(ex.name)) src = PurePosixPath(src) dst = Path(dst) if dst else Path(src.name) @@ -292,12 +409,6 @@ async def write( """ Write a file to a remote node using the standard Cyphal file service. """ - try: - from pycyphal.application.file import FileClient2 - except ImportError as ex: - from yakut.cmd.compile import make_usage_suggestion - - raise click.ClickException(make_usage_suggestion(ex.name)) src = Path(src) dst = PurePosixPath(dst) if dst else PurePosixPath(src.name) From 8da2e40ea991568afef33392ccb412ea1d5139d7 Mon Sep 17 00:00:00 2001 From: chemicstry Date: Tue, 20 May 2025 13:49:28 +0300 Subject: [PATCH 6/7] Fix formatting --- tests/cmd/file_client.py | 8 ++++++-- yakut/cmd/file_client/_cmd.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/cmd/file_client.py b/tests/cmd/file_client.py index 36f425c..9ba207b 100644 --- a/tests/cmd/file_client.py +++ b/tests/cmd/file_client.py @@ -135,7 +135,9 @@ async def _unittest_file_client_error_cases() -> None: server_root, client_root, srv_proc = await _setup_test_env() try: # Try to read non-existent file - exitcode, _, stderr = await _run_client_command("read", "42", "/nonexistent.txt", str(Path(client_root) / "local.txt")) + exitcode, _, stderr = await _run_client_command( + "read", "42", "/nonexistent.txt", str(Path(client_root) / "local.txt") + ) assert exitcode == EXIT_CODE_UNSUCCESSFUL assert "not found" in stderr.lower() @@ -157,7 +159,9 @@ async def _unittest_file_client_error_cases() -> None: assert exitcode == EXIT_CODE_UNSUCCESSFUL # Try to write with invalid node ID - exitcode, _, stderr = await _run_client_command("write", "999", "/test.txt", str(Path(client_root) / "local.txt")) + exitcode, _, stderr = await _run_client_command( + "write", "999", "/test.txt", str(Path(client_root) / "local.txt") + ) assert exitcode == EXIT_CODE_UNSUCCESSFUL finally: diff --git a/yakut/cmd/file_client/_cmd.py b/yakut/cmd/file_client/_cmd.py index 9251e35..033ac12 100644 --- a/yakut/cmd/file_client/_cmd.py +++ b/yakut/cmd/file_client/_cmd.py @@ -15,6 +15,7 @@ from pycyphal.application.file import FileClient2 import dataclasses + @dataclasses.dataclass class FileInfo: size: int From 281ccb9a3711ebef24e9711764ab300fd48e2c52 Mon Sep 17 00:00:00 2001 From: chemicstry Date: Tue, 20 May 2025 13:52:46 +0300 Subject: [PATCH 7/7] Move pycyphal imports --- yakut/cmd/file_client/_cmd.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/yakut/cmd/file_client/_cmd.py b/yakut/cmd/file_client/_cmd.py index 033ac12..1bb8dae 100644 --- a/yakut/cmd/file_client/_cmd.py +++ b/yakut/cmd/file_client/_cmd.py @@ -12,7 +12,6 @@ from yakut.param.formatter import FormatterHints from yakut.ui import ProgressReporter, show_error, show_warning from yakut.util import EXIT_CODE_UNSUCCESSFUL -from pycyphal.application.file import FileClient2 import dataclasses @@ -85,6 +84,7 @@ async def ls( """ List files on a remote node using the standard Cyphal file service. """ + from pycyphal.application.file import FileClient2 _logger.debug("node_ids=%r, path=%r, timeout=%r", node_ids, path, timeout) node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] @@ -165,6 +165,7 @@ async def mv( """ Move/rename a file or directory on remote node(s) using the standard Cyphal file service. """ + from pycyphal.application.file import FileClient2 _logger.debug("node_ids=%r, src=%r, dst=%r, timeout=%r", node_ids, src, dst, timeout) node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] @@ -214,6 +215,7 @@ async def cp( """ Copy a file on remote node(s) using the standard Cyphal file service. """ + from pycyphal.application.file import FileClient2 _logger.debug("node_ids=%r, src=%r, dst=%r, timeout=%r", node_ids, src, dst, timeout) node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] @@ -261,6 +263,7 @@ async def touch( """ Create an empty file or update timestamp on remote node(s) using the standard Cyphal file service. """ + from pycyphal.application.file import FileClient2 _logger.debug("node_ids=%r, path=%r, timeout=%r", node_ids, path, timeout) node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] @@ -306,6 +309,7 @@ async def rm( """ Remove a file or directory on remote node(s) using the standard Cyphal file service. """ + from pycyphal.application.file import FileClient2 _logger.debug("node_ids=%r, path=%r, timeout=%r", node_ids, path, timeout) node_ids_list = list(sorted(node_ids)) if isinstance(node_ids, set) else [node_ids] @@ -355,6 +359,7 @@ async def read( """ Read a file from a remote node using the standard Cyphal file service. """ + from pycyphal.application.file import FileClient2 src = PurePosixPath(src) dst = Path(dst) if dst else Path(src.name) @@ -410,6 +415,7 @@ async def write( """ Write a file to a remote node using the standard Cyphal file service. """ + from pycyphal.application.file import FileClient2 src = Path(src) dst = PurePosixPath(dst) if dst else PurePosixPath(src.name)