From 1cca9c07d70aae764104ad2eb9380aea4d74140b Mon Sep 17 00:00:00 2001 From: Lorenzo Maffioli Date: Fri, 5 Dec 2025 14:24:58 +0100 Subject: [PATCH 1/3] feat: deal with utf-16 and respect .gitignore --- .github/workflows/test-workflow.yml | 2 +- .pre-commit-config.yaml | 1 - isops/__init__.py | 7 +- isops/cli.py | 43 +++++-- isops/utils/__init__.py | 4 + isops/utils/helpers.py | 127 ++++++++++++++++++-- poetry.lock | 99 ++++++++++------ pyproject.toml | 2 +- tests/conftest.py | 12 ++ tests/samples/simple_secret_utf16.yaml | Bin 0 -> 320 bytes tests/samples/simple_secret_utf16_enc.yaml | Bin 0 -> 780 bytes tests/test_helpers.py | 130 ++++++++++++++++++++- tox.ini | 2 +- 13 files changed, 367 insertions(+), 62 deletions(-) create mode 100644 tests/samples/simple_secret_utf16.yaml create mode 100644 tests/samples/simple_secret_utf16_enc.yaml diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml index 5ca1b6b..a8aac20 100644 --- a/.github/workflows/test-workflow.yml +++ b/.github/workflows/test-workflow.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 223e259..36f46e9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,7 +30,6 @@ repos: additional_dependencies: - flake8-bugbear - flake8-comprehensions - - flake8-simplify - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.10.0 diff --git a/isops/__init__.py b/isops/__init__.py index 2e7a9d5..04b5c94 100644 --- a/isops/__init__.py +++ b/isops/__init__.py @@ -1,3 +1,6 @@ -import pkg_resources +try: + from importlib.metadata import version +except ImportError: + from importlib_metadata import version # type: ignore[import-not-found,no-redef] -__version__ = pkg_resources.get_distribution("isops").version +__version__ = version("isops") diff --git a/isops/cli.py b/isops/cli.py index 1d098f1..e16c006 100644 --- a/isops/cli.py +++ b/isops/cli.py @@ -1,6 +1,6 @@ import re from pathlib import Path -from typing import Dict, List, Pattern, Tuple +from typing import Dict, List, Optional, Pattern, Tuple import click @@ -10,6 +10,7 @@ find_all_files_by_regex, find_by_key, load_all_yaml, + load_all_yaml_with_encoding, verify_encryption_regex, ) @@ -31,6 +32,27 @@ def _categorize_keys_based_on_their_values( return good_keys, bad_keys +def _print_status(file: Path, key: str, is_safe: bool, encoding: Optional[str]) -> None: + """Print status line with optional encoding warning. + + Args: + file: The file path being checked. + key: The secret key name. + is_safe: Whether the secret is safely encrypted. + encoding: The detected file encoding, or None. + """ + status = "[SAFE]" if is_safe else "[UNSAFE]" + color = "green" if is_safe else "red" + + click.secho(message=f"{file}::{key} ", bold=False, nl=False) + click.secho(message=status, bold=False, fg=color, nl=False) + + if encoding and encoding.startswith("utf-16"): + click.secho(message=" [UTF-16 ENCODING]", bold=False, fg="yellow") + else: + click.echo() # Just newline + + def _validate_regex(ctx: click.Context, param: click.Parameter, value: str) -> str: try: re.compile(value) @@ -70,6 +92,9 @@ def cli(ctx: click.Context, path: Path, config_regex: Pattern[str], summary: boo creation_rules = [] for match_path in find_all_files_by_regex(config_regex, received_path): for config in load_all_yaml(Path(match_path)): + # Skip None (empty YAML documents) + if config is None: + continue try: creation_rules += config["creation_rules"] click.secho(message=f"Found config file: {match_path}", bold=True, fg="blue") @@ -132,12 +157,18 @@ def cli(ctx: click.Context, path: Path, config_regex: Pattern[str], summary: boo break for file in find_all_files_by_regex(path_regex, received_path): - if not load_all_yaml(file): + yaml_data, encoding = load_all_yaml_with_encoding(file) + + if not yaml_data: click.secho(message=f"{file} is not a valid YAML!", bold=True, fg="red") broken_yaml_found = f"{file}" break - for secret in load_all_yaml(file): + for secret in yaml_data: + # Skip None (empty YAML documents) + if secret is None: + continue + if "sops" in secret: secret.pop("sops", None) @@ -148,12 +179,10 @@ def cli(ctx: click.Context, path: Path, config_regex: Pattern[str], summary: boo for key in all_keys: if key in good_keys: - click.secho(message=f"{file}::{key} ", bold=False, nl=False) - click.secho(message="[SAFE]", bold=False, fg="green") + _print_status(file, key, True, encoding) good_keys_number += 1 else: - click.secho(message=f"{file}::{key} ", bold=False, nl=False) - click.secho(message="[UNSAFE]", bold=False, fg="red") + _print_status(file, key, False, encoding) bad_keys_number += 1 if summary: summary_line = f"UNSAFE secret '{key}' in '{file}'" diff --git a/isops/utils/__init__.py b/isops/utils/__init__.py index a450c1c..3c6a4c5 100644 --- a/isops/utils/__init__.py +++ b/isops/utils/__init__.py @@ -1,8 +1,10 @@ from isops.utils.helpers import ( all_dict_values, + detect_encoding, find_all_files_by_regex, find_by_key, load_all_yaml, + load_all_yaml_with_encoding, load_yaml, ) from isops.utils.sops import verify_encryption_regex @@ -10,6 +12,8 @@ __all__ = [ "load_yaml", "load_all_yaml", + "load_all_yaml_with_encoding", + "detect_encoding", "find_by_key", "all_dict_values", "verify_encryption_regex", diff --git a/isops/utils/helpers.py b/isops/utils/helpers.py index ec8e8a9..41dff3c 100644 --- a/isops/utils/helpers.py +++ b/isops/utils/helpers.py @@ -1,13 +1,45 @@ import os import re from pathlib import Path -from typing import Dict, Generator, List, Pattern, Tuple +from typing import Dict, Generator, List, Optional, Pattern, Tuple +import pathspec from ruamel.yaml import YAML, YAMLError from ruamel.yaml.parser import ParserError from ruamel.yaml.scanner import ScannerError +def detect_encoding(path: Path) -> Optional[str]: + """Detect the encoding of a file using BOM markers. + + Checks for UTF-16 BOM markers. Defaults to UTF-8 for files without BOM. + Only reads a sample of the file for performance. + + Args: + path (Path): The path of the file. + + Returns: + Optional[str]: 'utf-16-le', 'utf-16-be', 'utf-8', or None on error. + """ + try: + with open(path, "rb") as f: + # Read first 2 bytes for BOM check + bom = f.read(2) + if len(bom) < 2: + return None + + # Check for UTF-16 BOM markers + if bom == b"\xfe\xff": + return "utf-16-be" + elif bom == b"\xff\xfe": + return "utf-16-le" + + # Default to UTF-8 for files without BOM + return "utf-8" + except OSError: + return None + + def load_yaml(path: Path) -> Dict: """Load a YAML content into a python dictionary. @@ -20,7 +52,7 @@ def load_yaml(path: Path) -> Dict: try: yaml = YAML(typ="safe") return yaml.load(path) - except YAMLError: + except (YAMLError, UnicodeDecodeError): return {} @@ -38,10 +70,38 @@ def load_all_yaml(path: Path) -> List[Dict]: try: yaml = YAML(typ="safe") return list(yaml.load_all(path)) - except (ParserError, ScannerError): + except (ParserError, ScannerError, UnicodeDecodeError): return [] +def load_all_yaml_with_encoding(path: Path) -> Tuple[List[Dict], Optional[str]]: + """Like load_all_yaml, but also returns the detected encoding. + + Args: + path (Path): The path of the YAML file. + + Returns: + Tuple[List[Dict], Optional[str]]: A tuple containing: + - List of dictionaries corresponding to the different yaml blocks + - The detected encoding (e.g., 'utf-8', 'utf-16') or None + If parsing fails or file cannot be read, returns ([], None). + """ + encoding = detect_encoding(path) + + # Handle UTF-16 files explicitly + if encoding and encoding.startswith("utf-16"): + try: + yaml = YAML(typ="safe") + with open(path, "r", encoding=encoding) as f: + return list(yaml.load_all(f)), encoding + except (ParserError, ScannerError, UnicodeDecodeError, OSError): + return [], None + + # For UTF-8 or errors, use standard load_all_yaml + data = load_all_yaml(path) + return data, encoding if data else None + + def find_by_key(data: Dict, target: Pattern[str]) -> Generator[Dict, None, None]: """Find the innermost key-value pair children of a target key in a dictionary. @@ -82,25 +142,72 @@ def all_dict_values(data: Dict) -> Generator[Tuple[str, str], None, None]: yield from all_dict_values(value) elif isinstance(value, list): for elem in value: - yield from all_dict_values(elem) + # Only recurse if the element is a dict + if isinstance(elem, dict): + yield from all_dict_values(elem) elif not isinstance(value, dict): yield key, str(value) +def _load_gitignore_spec(search_path: Path) -> Optional[pathspec.PathSpec]: + """Load .gitignore patterns from the search path. + + Args: + search_path (Path): The root path to search for .gitignore. + + Returns: + Optional[pathspec.PathSpec]: A PathSpec object, or None if no .gitignore found + or if parsing fails. + """ + gitignore_path = search_path / ".gitignore" + if not (gitignore_path.exists() and gitignore_path.is_file()): + return None + + try: + with open(gitignore_path, "r", encoding="utf-8") as f: + patterns = f.read().splitlines() + return pathspec.PathSpec.from_lines("gitwildmatch", patterns) + except (OSError, UnicodeDecodeError): + # File access or encoding issues - return None + return None + + def find_all_files_by_regex(regex: Pattern[str], path: Path) -> Generator[Path, None, None]: """Find all the files that match a regular expression. + Respects .gitignore patterns if a .gitignore file exists in the search path. + Automatically excludes .git directory. + Args: - regex (Pattern[str]): Regex pattern. + regex (Pattern[str]): Regex pattern (string or compiled). path (Path): Path of the root directory to search. Yields: Generator[Path, None, None]: Iterable of all the files in 'path' that match the 'regex'. """ - pattern: Pattern[str] = re.compile(regex) - for root, _, files in os.walk(path): + # Ensure pattern is compiled (handles both string and Pattern inputs) + pattern = re.compile(regex) if isinstance(regex, str) else regex + gitignore_spec = _load_gitignore_spec(path) + + for root, dirs, files in os.walk(path): + root_path = Path(root) + rel_root = root_path.relative_to(path) + + # Filter directories: exclude .git and gitignored dirs in one pass + dirs[:] = [ + d + for d in dirs + if d != ".git" + and (not gitignore_spec or not gitignore_spec.match_file(str(rel_root / d) + "/")) + ] + for file in files: - match = pattern.search(os.path.join(root, file)) - if match: - yield Path(os.path.join(root, file)) + file_path = root_path / file + + # Check if file matches the regex and is not ignored + if pattern.search(str(file_path)): + if not gitignore_spec or not gitignore_spec.match_file( + str(file_path.relative_to(path)) + ): + yield file_path diff --git a/poetry.lock b/poetry.lock index 52de805..e194852 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,15 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. - -[[package]] -name = "astor" -version = "0.8.1" -description = "Read/rewrite/write Python ASTs" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" -files = [ - {file = "astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5"}, - {file = "astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e"}, -] +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "attrs" @@ -17,6 +6,7 @@ version = "23.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, @@ -27,8 +17,8 @@ cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] dev = ["attrs[tests]", "pre-commit"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] -tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.6) ; platform_python_implementation == \"CPython\" and python_version >= \"3.8\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.8\""] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] [[package]] name = "black" @@ -36,6 +26,7 @@ version = "24.4.2" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "black-24.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dd1b5a14e417189db4c7b64a6540f31730713d173f0b63e55fabd52d61d8fdce"}, {file = "black-24.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e537d281831ad0e71007dcdcbe50a71470b978c453fa41ce77186bbe0ed6021"}, @@ -72,7 +63,7 @@ typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +d = ["aiohttp (>=3.7.4) ; sys_platform != \"win32\" or implementation_name != \"pypy\"", "aiohttp (>=3.7.4,!=3.9.0) ; sys_platform == \"win32\" and implementation_name == \"pypy\""] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] @@ -82,6 +73,7 @@ version = "3.4.0" description = "Validate configuration and produce human readable error messages." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, @@ -93,6 +85,7 @@ version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, @@ -107,10 +100,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "dev"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "platform_system == \"Windows\"", dev = "platform_system == \"Windows\" or sys_platform == \"win32\""} [[package]] name = "coverage" @@ -118,6 +113,7 @@ version = "7.5.1" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "coverage-7.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0884920835a033b78d1c73b6d3bbcda8161a900f38a488829a83982925f6c2e"}, {file = "coverage-7.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:39afcd3d4339329c5f58de48a52f6e4e50f6578dd6099961cf22228feb25f38f"}, @@ -177,7 +173,7 @@ files = [ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} [package.extras] -toml = ["tomli"] +toml = ["tomli ; python_full_version <= \"3.11.0a6\""] [[package]] name = "distlib" @@ -185,6 +181,7 @@ version = "0.3.8" description = "Distribution utilities" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, @@ -196,6 +193,8 @@ version = "1.2.1" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "python_version < \"3.11\"" files = [ {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, @@ -210,6 +209,7 @@ version = "3.14.0" description = "A platform independent file lock." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "filelock-3.14.0-py3-none-any.whl", hash = "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f"}, {file = "filelock-3.14.0.tar.gz", hash = "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a"}, @@ -218,7 +218,7 @@ files = [ [package.extras] docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] -typing = ["typing-extensions (>=4.8)"] +typing = ["typing-extensions (>=4.8) ; python_version < \"3.11\""] [[package]] name = "flake8" @@ -226,6 +226,7 @@ version = "7.0.0" description = "the modular source code checker: pep8 pyflakes and co" optional = false python-versions = ">=3.8.1" +groups = ["dev"] files = [ {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"}, {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"}, @@ -242,6 +243,7 @@ version = "24.4.26" description = "A plugin for flake8 finding likely bugs and design problems in your program. Contains warnings that don't belong in pyflakes and pycodestyle." optional = false python-versions = ">=3.8.1" +groups = ["dev"] files = [ {file = "flake8_bugbear-24.4.26-py3-none-any.whl", hash = "sha256:cb430dd86bc821d79ccc0b030789a9c87a47a369667f12ba06e80f11305e8258"}, {file = "flake8_bugbear-24.4.26.tar.gz", hash = "sha256:ff8d4ba5719019ebf98e754624c30c05cef0dadcf18a65d91c7567300e52a130"}, @@ -260,6 +262,7 @@ version = "3.14.0" description = "A flake8 plugin to help you write better list/set/dict comprehensions." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "flake8_comprehensions-3.14.0-py3-none-any.whl", hash = "sha256:7b9d07d94aa88e62099a6d1931ddf16c344d4157deedf90fe0d8ee2846f30e97"}, {file = "flake8_comprehensions-3.14.0.tar.gz", hash = "sha256:81768c61bfc064e1a06222df08a2580d97de10cb388694becaf987c331c6c0cf"}, @@ -268,27 +271,13 @@ files = [ [package.dependencies] flake8 = ">=3.0,<3.2.0 || >3.2.0" -[[package]] -name = "flake8-simplify" -version = "0.21.0" -description = "flake8 plugin which checks for code that can be simplified" -optional = false -python-versions = ">=3.6.1" -files = [ - {file = "flake8_simplify-0.21.0-py3-none-any.whl", hash = "sha256:439391e762a9370b371208add0b5c5c40c3d25a98e1f5421d263215d08194183"}, - {file = "flake8_simplify-0.21.0.tar.gz", hash = "sha256:c95ff1dcc1de5949af47e0087cbf1164445881131b15bcd7a71252670f492f4d"}, -] - -[package.dependencies] -astor = ">=0.1" -flake8 = ">=3.7" - [[package]] name = "identify" version = "2.5.36" description = "File identification library for Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "identify-2.5.36-py2.py3-none-any.whl", hash = "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa"}, {file = "identify-2.5.36.tar.gz", hash = "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d"}, @@ -303,6 +292,7 @@ version = "2.0.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, @@ -314,6 +304,7 @@ version = "5.13.2" description = "A Python utility / library to sort Python imports." optional = false python-versions = ">=3.8.0" +groups = ["dev"] files = [ {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, @@ -328,6 +319,7 @@ version = "0.7.0" description = "McCabe checker, plugin for flake8" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, @@ -339,6 +331,7 @@ version = "1.10.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mypy-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2"}, {file = "mypy-1.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99"}, @@ -386,6 +379,7 @@ version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.5" +groups = ["dev"] files = [ {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, @@ -397,6 +391,7 @@ version = "1.8.0" description = "Node.js virtual environment builder" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +groups = ["dev"] files = [ {file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"}, {file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"}, @@ -411,6 +406,7 @@ version = "24.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, @@ -422,6 +418,7 @@ version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, @@ -433,6 +430,7 @@ version = "4.2.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "platformdirs-4.2.1-py3-none-any.whl", hash = "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1"}, {file = "platformdirs-4.2.1.tar.gz", hash = "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf"}, @@ -449,6 +447,7 @@ version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -464,6 +463,7 @@ version = "3.7.1" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pre_commit-3.7.1-py2.py3-none-any.whl", hash = "sha256:fae36fd1d7ad7d6a5a1c0b0d5adb2ed1a3bda5a21bf6c3e5372073d7a11cd4c5"}, {file = "pre_commit-3.7.1.tar.gz", hash = "sha256:8ca3ad567bc78a4972a3f1a477e94a79d4597e8140a6e0b651c5e33899c3654a"}, @@ -482,6 +482,7 @@ version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["dev"] files = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, @@ -493,6 +494,7 @@ version = "2.11.1" description = "Python style guide checker" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, @@ -504,6 +506,7 @@ version = "6.3.0" description = "Python docstring style checker" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "pydocstyle-6.3.0-py3-none-any.whl", hash = "sha256:118762d452a49d6b05e194ef344a55822987a462831ade91ec5c06fd2169d019"}, {file = "pydocstyle-6.3.0.tar.gz", hash = "sha256:7ce43f0c0ac87b07494eb9c0b462c0b73e6ff276807f204d6b53edc72b7e44e1"}, @@ -513,7 +516,7 @@ files = [ snowballstemmer = ">=2.2.0" [package.extras] -toml = ["tomli (>=1.2.3)"] +toml = ["tomli (>=1.2.3) ; python_version < \"3.11\""] [[package]] name = "pyflakes" @@ -521,6 +524,7 @@ version = "3.2.0" description = "passive checker of Python programs" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"}, {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"}, @@ -532,6 +536,7 @@ version = "8.2.0" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pytest-8.2.0-py3-none-any.whl", hash = "sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233"}, {file = "pytest-8.2.0.tar.gz", hash = "sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f"}, @@ -554,6 +559,7 @@ version = "5.0.0" description = "Pytest plugin for measuring coverage." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pytest-cov-5.0.0.tar.gz", hash = "sha256:5837b58e9f6ebd335b0f8060eecce69b662415b16dc503883a02f45dfeb14857"}, {file = "pytest_cov-5.0.0-py3-none-any.whl", hash = "sha256:4f0764a1219df53214206bf1feea4633c3b558a2925c8b59f144f682861ce652"}, @@ -572,6 +578,7 @@ version = "6.0.1" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, @@ -632,6 +639,7 @@ version = "0.18.6" description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "ruamel.yaml-0.18.6-py3-none-any.whl", hash = "sha256:57b53ba33def16c4f3d807c0ccbc00f8a6081827e81ba2491691b76882d0c636"}, {file = "ruamel.yaml-0.18.6.tar.gz", hash = "sha256:8b27e6a217e786c6fbe5634d8f3f11bc63e0f80f6a5890f28863d9c45aac311b"}, @@ -650,6 +658,8 @@ version = "0.2.8" description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" optional = false python-versions = ">=3.6" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version < \"3.13\"" files = [ {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462"}, @@ -709,6 +719,7 @@ version = "69.5.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, @@ -716,7 +727,7 @@ files = [ [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov ; platform_python_implementation != \"PyPy\"", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -725,6 +736,7 @@ version = "1.16.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +groups = ["dev"] files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -736,6 +748,7 @@ version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, @@ -747,6 +760,7 @@ version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +groups = ["dev"] files = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -758,6 +772,8 @@ version = "2.0.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "python_full_version <= \"3.11.0a6\"" files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, @@ -769,6 +785,7 @@ version = "3.28.0" description = "tox is a generic virtualenv management and test command line tool" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +groups = ["dev"] files = [ {file = "tox-3.28.0-py2.py3-none-any.whl", hash = "sha256:57b5ab7e8bb3074edc3c0c0b4b192a4f3799d3723b2c5b76f1fa9f2d40316eea"}, {file = "tox-3.28.0.tar.gz", hash = "sha256:d0d28f3fe6d6d7195c27f8b054c3e99d5451952b54abdae673b71609a581f640"}, @@ -786,7 +803,7 @@ virtualenv = ">=16.0.0,<20.0.0 || >20.0.0,<20.0.1 || >20.0.1,<20.0.2 || >20.0.2, [package.extras] docs = ["pygments-github-lexers (>=0.0.5)", "sphinx (>=2.0.0)", "sphinxcontrib-autoprogram (>=0.1.5)", "towncrier (>=18.5.0)"] -testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pathlib2 (>=2.3.3)", "psutil (>=5.6.1)", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)"] +testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pathlib2 (>=2.3.3) ; python_version < \"3.4\"", "psutil (>=5.6.1) ; platform_python_implementation == \"cpython\"", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)"] [[package]] name = "tox-pyenv" @@ -794,6 +811,7 @@ version = "1.1.0" description = "tox plugin that makes tox use `pyenv which` to find python executables" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "tox-pyenv-1.1.0.tar.gz", hash = "sha256:916c2213577aec0b3b5452c5bfb32fd077f3a3196f50a81ad57d7ef3fc2599e4"}, {file = "tox_pyenv-1.1.0-py2.py3-none-any.whl", hash = "sha256:e470c18af115fe52eeff95e7e3cdd0793613eca19709966fc2724b79d55246cb"}, @@ -808,6 +826,7 @@ version = "0.21.0.20240423" description = "Typing stubs for docutils" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-docutils-0.21.0.20240423.tar.gz", hash = "sha256:7716ec6c68b5179b7ba1738cace2f1326e64df9f44b7ab08d9904d32c23fc15f"}, {file = "types_docutils-0.21.0.20240423-py3-none-any.whl", hash = "sha256:7f6e84ba8fcd2454c5b8bb8d77384d091a901929cc2b31079316e10eb346580a"}, @@ -819,6 +838,7 @@ version = "6.0.12.20240311" description = "Typing stubs for PyYAML" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-PyYAML-6.0.12.20240311.tar.gz", hash = "sha256:a9e0f0f88dc835739b0c1ca51ee90d04ca2a897a71af79de9aec5f38cb0a5342"}, {file = "types_PyYAML-6.0.12.20240311-py3-none-any.whl", hash = "sha256:b845b06a1c7e54b8e5b4c683043de0d9caf205e7434b3edc678ff2411979b8f6"}, @@ -830,6 +850,7 @@ version = "65.7.0.4" description = "Typing stubs for setuptools" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "types-setuptools-65.7.0.4.tar.gz", hash = "sha256:147809433301fe7e0f4ef5c0782f9a0453788960575e1efb6da5fe8cb2493c9f"}, {file = "types_setuptools-65.7.0.4-py3-none-any.whl", hash = "sha256:522067dfd8e1771f8d7e047e451de2740dc4e0c9f48a22302a6cc96e6c964a13"}, @@ -844,6 +865,7 @@ version = "4.11.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, @@ -855,6 +877,7 @@ version = "20.26.1" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "virtualenv-20.26.1-py3-none-any.whl", hash = "sha256:7aa9982a728ae5892558bff6a2839c00b9ed145523ece2274fad6f414690ae75"}, {file = "virtualenv-20.26.1.tar.gz", hash = "sha256:604bfdceaeece392802e6ae48e69cec49168b9c5f4a44e483963f9242eb0e78b"}, @@ -867,9 +890,9 @@ platformdirs = ">=3.9.1,<5" [package.extras] docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] -test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8) ; platform_python_implementation == \"PyPy\"", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10) ; platform_python_implementation == \"CPython\""] [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = "^3.9" -content-hash = "f08eb8d2ce2b45fe00d8f8b902f9a230eebeef2035dcbeb0a9f6fac5a9212646" +content-hash = "062ebd6fd76f9ab6d0a76958c344377fa6889f8726dcae67575f0b96dc545a7b" diff --git a/pyproject.toml b/pyproject.toml index 9271ff5..b41560a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ python = "^3.9" click = "^8.1.3" ruamel-yaml = "^0.18.0" setuptools = "^69.5.1" +pathspec = "^0.12.0" [tool.poetry.group.dev.dependencies] black = "^24.0.0" @@ -23,7 +24,6 @@ isort = "^5.10.1" pytest = "^8.0.0" pytest-cov = "^5.0.0" flake8 = "^7.0.0" -flake8-simplify = "^0.21.0" flake8-bugbear = "^24.0.0" flake8-comprehensions = "^3.10.1" mypy = "^1.0.0" diff --git a/tests/conftest.py b/tests/conftest.py index 9103fe2..15902ab 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -88,6 +88,18 @@ def yaml_blocks(): return load_all_yaml(path) +@pytest.fixture(scope="module") +def simple_secret_utf16_yaml(): + path = Path(os.path.join(SAMPLES_PATH, "simple_secret_utf16.yaml")) + return load_yaml(path) + + +@pytest.fixture(scope="module") +def simple_secret_utf16_enc_yaml(): + path = Path(os.path.join(SAMPLES_PATH, "simple_secret_utf16_enc.yaml")) + return load_yaml(path) + + @pytest.fixture(scope="function") def simple_dir_struct(tmp_path, example_dotspos_yaml): """Nobody forbids me to make a fixture that returns a function""" diff --git a/tests/samples/simple_secret_utf16.yaml b/tests/samples/simple_secret_utf16.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc714ddd7eb38a0a14db8f46f4037ff26529d44e GIT binary patch literal 320 zcmZvY%L>9U5Jm60U(p|Mp@@ruJ8`E%MN8ck+k$8xwvFP?t0y&uE+mA^WX{~V$>*J@ z)33C&~?amfk?e zdg-K<*1C}wI%uXXJ0$jW#YyStJu}YqX6SqUMC<6Ev$3NBed}}RoXF6h^Qy(j-O$nU b3kmG+H7RFKZt!Mjc4*bEw(H3|wT#LaNk}zU literal 0 HcmV?d00001 diff --git a/tests/samples/simple_secret_utf16_enc.yaml b/tests/samples/simple_secret_utf16_enc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6501c51de5f601f013fbdfe8d0d4a7434fc3e7e4 GIT binary patch literal 780 zcmb7?TTjA35QWb(zk+umTtq|?9`FL9ARv@#j4`Evaur$-@W-p)7Em6HiP>y-J2QJ` z&YAiAa5Pn4Ep_F|=k}_JvI^@>35C>=qd7avJ-l5D;!N}e`uqJh)no?@AQ_2z3Cu3ISz~_S2B7cdv3NeS&cnfk{ww9VHBMY1c zIg+55d}ZqCVtdhwo2M$BhTq8lJjYckY z8SJupTIi10A=Ow!4LD2S=0U3yv;0G&Aw8XJHEE!q3!5%ELc!FSe*89F`%^WhQ)bq7 wFQZg5r=|pTN3olZ{mkX=r4Jtm&t@_^oIU;*C}48W!Q6B+jPfROhOdj~3;HU4vH$=8 literal 0 HcmV?d00001 diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 23cb559..5284ee5 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,7 +1,20 @@ +import os +from pathlib import Path + import pytest from ruamel.yaml import YAML -from isops.utils import all_dict_values, find_by_key, load_all_yaml +from isops.utils import ( + all_dict_values, + detect_encoding, + find_all_files_by_regex, + find_by_key, + load_all_yaml, + load_all_yaml_with_encoding, +) + +TESTS_PATH = os.path.dirname(os.path.realpath(__file__)) +SAMPLES_PATH = os.path.join(TESTS_PATH, "samples") def test_is_yaml_loaded_correctly(example_good_deploy_yaml): @@ -176,3 +189,118 @@ def test_all_dict_values_nested_messy_yaml(): for _, val in all_dict_values(input): got.append(val) assert got == expected + + +def test_utf16_yaml_loaded_correctly(simple_secret_utf16_yaml): + """Test that UTF-16 encoded YAML files are loaded correctly""" + expected = { + "apiVersion": "v1", + "data": {"username": "YWRtaW4=", "password": "MWYyZDFlMmU2N2Rm"}, + "kind": "Secret", + "metadata": {"name": "mysecret-utf16", "namespace": "default"}, + "type": "Opaque", + } + assert isinstance(simple_secret_utf16_yaml, dict) + assert simple_secret_utf16_yaml == expected + + +def test_utf16_encoding_detection(): + """Test that UTF-16 encoding is correctly detected""" + path = Path(os.path.join(SAMPLES_PATH, "simple_secret_utf16.yaml")) + encoding = detect_encoding(path) + assert encoding is not None + assert encoding.startswith("utf-16") + + +def test_load_all_yaml_with_encoding_utf16(): + """Test that load_all_yaml_with_encoding returns correct data and encoding for UTF-16 files""" + path = Path(os.path.join(SAMPLES_PATH, "simple_secret_utf16.yaml")) + data, encoding = load_all_yaml_with_encoding(path) + + assert len(data) == 1 + assert encoding is not None + assert encoding.startswith("utf-16") + assert data[0]["kind"] == "Secret" + assert data[0]["metadata"]["name"] == "mysecret-utf16" + + +def test_load_all_yaml_with_encoding_utf8(): + """Test that load_all_yaml_with_encoding returns correct data and encoding for UTF-8 files""" + path = Path(os.path.join(SAMPLES_PATH, "simple_secret.yaml")) + data, encoding = load_all_yaml_with_encoding(path) + + assert len(data) == 1 + assert encoding == "utf-8" + assert data[0]["kind"] == "Secret" + + +def test_utf16_encrypted_yaml_loaded_correctly(simple_secret_utf16_enc_yaml): + """Test that UTF-16 encoded encrypted YAML files are loaded correctly""" + assert isinstance(simple_secret_utf16_enc_yaml, dict) + assert simple_secret_utf16_enc_yaml["kind"] == "Secret" + assert simple_secret_utf16_enc_yaml["metadata"]["name"] == "mysecret-utf16-enc" + # Check that encrypted values start with ENC[ + assert simple_secret_utf16_enc_yaml["data"]["username"].startswith("ENC[") + assert simple_secret_utf16_enc_yaml["data"]["password"].startswith("ENC[") + + +def test_find_all_files_respects_gitignore(tmp_path): + """Test that find_all_files_by_regex respects .gitignore patterns""" + # Create test directory structure + (tmp_path / "included").mkdir() + (tmp_path / "ignored_dir").mkdir() + (tmp_path / "nested" / "ignored_nested").mkdir(parents=True) + + # Create test files + (tmp_path / "file1.yaml").write_text("test: 1") + (tmp_path / "file2.yaml").write_text("test: 2") + (tmp_path / "included" / "file3.yaml").write_text("test: 3") + (tmp_path / "ignored_dir" / "file4.yaml").write_text("test: 4") + (tmp_path / "ignored_file.yaml").write_text("test: ignored") + (tmp_path / "nested" / "file5.yaml").write_text("test: 5") + (tmp_path / "nested" / "ignored_nested" / "file6.yaml").write_text("test: 6") + + # Create .gitignore + gitignore_content = """# Ignore specific directory +ignored_dir/ +# Ignore specific file +ignored_file.yaml +# Ignore nested directory +nested/ignored_nested/ +""" + (tmp_path / ".gitignore").write_text(gitignore_content) + + # Find all YAML files + found_files = list(find_all_files_by_regex(r"\.yaml$", tmp_path)) + found_names = {f.name for f in found_files} + + # Should find files that are not ignored + assert "file1.yaml" in found_names + assert "file2.yaml" in found_names + assert "file3.yaml" in found_names + assert "file5.yaml" in found_names + assert ".gitignore" not in found_names + + # Should NOT find ignored files + assert "ignored_file.yaml" not in found_names + assert "file4.yaml" not in found_names # in ignored_dir + assert "file6.yaml" not in found_names # in ignored_nested + + +def test_find_all_files_without_gitignore(tmp_path): + """Test that find_all_files_by_regex works without .gitignore""" + # Create test files + (tmp_path / "file1.yaml").write_text("test: 1") + (tmp_path / "file2.yaml").write_text("test: 2") + (tmp_path / "subdir").mkdir() + (tmp_path / "subdir" / "file3.yaml").write_text("test: 3") + + # Find all YAML files (no .gitignore exists) + found_files = list(find_all_files_by_regex(r"\.yaml$", tmp_path)) + found_names = {f.name for f in found_files} + + # Should find all files + assert len(found_names) == 3 + assert "file1.yaml" in found_names + assert "file2.yaml" in found_names + assert "file3.yaml" in found_names diff --git a/tox.ini b/tox.ini index 386de38..580a506 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py312, py311, py310, py39 +envlist = py314, py313,py312, py311, py310, py39 isolated_build = True [testenv] From 5192bdd5066c45badce3a978ed3d475b66f95a13 Mon Sep 17 00:00:00 2001 From: Lorenzo Maffioli Date: Fri, 5 Dec 2025 14:30:20 +0100 Subject: [PATCH 2/3] poetry lock --- poetry.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index e194852..d3b17d4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -418,7 +418,7 @@ version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, @@ -895,4 +895,4 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.1" python-versions = "^3.9" -content-hash = "062ebd6fd76f9ab6d0a76958c344377fa6889f8726dcae67575f0b96dc545a7b" +content-hash = "73e6469fd3bd0d904f8ee0a5c92f00b039a0524b2055e76ddc93445c4fde7703" From e8a6b6d291fd476916f24bf3da8f74e7223e5eb5 Mon Sep 17 00:00:00 2001 From: Lorenzo Maffioli Date: Fri, 5 Dec 2025 14:33:48 +0100 Subject: [PATCH 3/3] updated poetry version in workflow --- .github/workflows/test-workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml index a8aac20..69c7bfe 100644 --- a/.github/workflows/test-workflow.yml +++ b/.github/workflows/test-workflow.yml @@ -25,10 +25,10 @@ jobs: uses: actions/cache@v4 with: path: ~/.local - key: poetry-1.2.2-0 + key: poetry-2.0.0-0 - uses: snok/install-poetry@v1 with: - version: 1.2.2 + version: 2.0.0 virtualenvs-create: true virtualenvs-in-project: true - name: Install dependencies @@ -50,7 +50,7 @@ jobs: python-version: '3.11' - uses: snok/install-poetry@v1 with: - version: 1.2.2 + version: 2.0.0 virtualenvs-create: true virtualenvs-in-project: true - name: Install dependencies