From 74536d907e90df6d3146a6bf813697a8c236fab9 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 5 Aug 2025 12:45:11 +0200 Subject: [PATCH 1/2] Restructure tests folder --- .gitattributes | 1 + .gitignore | 4 +- .../selectors.py | 0 pyproject.toml | 2 +- tests/_data/.gitkeep | 0 tests/{docs => _docs}/Makefile | 0 tests/{docs => _docs}/conf.py | 0 tests/{docs => _docs}/index.rst | 0 tests/adapters/__init__.py | 0 tests/{ => adapters}/test_avro.py | 51 +++++++++++++++- .../test_csv.py} | 0 .../test_elastic.py} | 0 .../test_json.py} | 3 +- .../test_line.py} | 0 .../test_splunk.py} | 0 .../test_sqlite_duckdb.py} | 0 .../test_text.py} | 0 .../test_xlsx.py} | 0 tests/fieldtypes/__init__.py | 0 tests/{ => fieldtypes}/test_fieldtypes.py | 0 .../test_ip.py} | 0 tests/packers/__init__.py | 0 tests/{ => packers}/test_json_packer.py | 0 tests/{ => packers}/test_packer.py | 0 tests/records/__init__.py | 0 .../test_adapter.py} | 3 +- .../test_descriptor.py} | 0 tests/{ => records}/test_multi_timestamp.py | 0 tests/{ => records}/test_record.py | 0 tests/selectors/__init__.py | 0 .../test_compiled.py} | 0 .../test_selectors.py} | 0 tests/standalone_test.py | 19 ------ tests/test_avro_adapter.py | 58 ------------------- ...test_regression.py => test_regressions.py} | 0 tests/tools/__init__.py | 0 tests/{ => tools}/test_rdump.py | 0 tox.ini | 8 +-- 38 files changed, 60 insertions(+), 89 deletions(-) create mode 100644 .gitattributes rename tests/selector_explain_example.py => examples/selectors.py (100%) create mode 100644 tests/_data/.gitkeep rename tests/{docs => _docs}/Makefile (100%) rename tests/{docs => _docs}/conf.py (100%) rename tests/{docs => _docs}/index.rst (100%) create mode 100644 tests/adapters/__init__.py rename tests/{ => adapters}/test_avro.py (55%) rename tests/{test_csv_adapter.py => adapters/test_csv.py} (100%) rename tests/{test_elastic_adapter.py => adapters/test_elastic.py} (100%) rename tests/{test_json_record_adapter.py => adapters/test_json.py} (98%) rename tests/{test_adapter_line.py => adapters/test_line.py} (100%) rename tests/{test_splunk_adapter.py => adapters/test_splunk.py} (100%) rename tests/{test_sqlite_duckdb_adapter.py => adapters/test_sqlite_duckdb.py} (100%) rename tests/{test_adapter_text.py => adapters/test_text.py} (100%) rename tests/{test_xlsx_adapter.py => adapters/test_xlsx.py} (100%) create mode 100644 tests/fieldtypes/__init__.py rename tests/{ => fieldtypes}/test_fieldtypes.py (100%) rename tests/{test_fieldtype_ip.py => fieldtypes/test_ip.py} (100%) create mode 100644 tests/packers/__init__.py rename tests/{ => packers}/test_json_packer.py (100%) rename tests/{ => packers}/test_packer.py (100%) create mode 100644 tests/records/__init__.py rename tests/{test_record_adapter.py => records/test_adapter.py} (99%) rename tests/{test_record_descriptor.py => records/test_descriptor.py} (100%) rename tests/{ => records}/test_multi_timestamp.py (100%) rename tests/{ => records}/test_record.py (100%) create mode 100644 tests/selectors/__init__.py rename tests/{test_compiled_selector.py => selectors/test_compiled.py} (100%) rename tests/{test_selector.py => selectors/test_selectors.py} (100%) delete mode 100644 tests/standalone_test.py delete mode 100644 tests/test_avro_adapter.py rename tests/{test_regression.py => test_regressions.py} (100%) create mode 100644 tests/tools/__init__.py rename tests/{ => tools}/test_rdump.py (100%) diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..9b4f63fa --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/_data/** filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index 9f8af7b2..975a9c46 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,6 @@ __pycache__/ .pytest_cache/ flow/record/version.py -tests/docs/api -tests/docs/build +tests/_docs/api +tests/_docs/build .tox/ diff --git a/tests/selector_explain_example.py b/examples/selectors.py similarity index 100% rename from tests/selector_explain_example.py rename to examples/selectors.py diff --git a/pyproject.toml b/pyproject.toml index bf38e5c8..b8c25b3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,7 +125,7 @@ select = [ ignore = ["E203", "B904", "UP024", "ANN002", "ANN003", "ANN204", "ANN401", "SIM105", "TRY003", "TRY400"] [tool.ruff.lint.per-file-ignores] -"tests/docs/**" = ["INP001"] +"tests/_docs/**" = ["INP001"] [tool.ruff.lint.isort] known-first-party = ["flow.record"] diff --git a/tests/_data/.gitkeep b/tests/_data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/docs/Makefile b/tests/_docs/Makefile similarity index 100% rename from tests/docs/Makefile rename to tests/_docs/Makefile diff --git a/tests/docs/conf.py b/tests/_docs/conf.py similarity index 100% rename from tests/docs/conf.py rename to tests/_docs/conf.py diff --git a/tests/docs/index.rst b/tests/_docs/index.rst similarity index 100% rename from tests/docs/index.rst rename to tests/_docs/index.rst diff --git a/tests/adapters/__init__.py b/tests/adapters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_avro.py b/tests/adapters/test_avro.py similarity index 55% rename from tests/test_avro.py rename to tests/adapters/test_avro.py index b8a6250f..c77f27d5 100644 --- a/tests/test_avro.py +++ b/tests/adapters/test_avro.py @@ -5,9 +5,10 @@ import pytest -from flow.record import RecordDescriptor, RecordReader +from flow.record import RecordDescriptor, RecordReader, RecordWriter from flow.record.adapter.avro import AvroReader, AvroWriter from flow.record.base import HAS_AVRO +from tests._utils import generate_plain_records if TYPE_CHECKING: from collections.abc import Iterator @@ -68,3 +69,51 @@ def test_avrostream_filelike_object(tmp_path: Path) -> None: assert rec.name == f"record{index}" assert rec.foo == "bar" assert rec.bar == "baz" + + +def test_avro_adapter(tmpdir: Path) -> None: + json_file = tmpdir.join("records.avro") + record_adapter_path = f"avro://{json_file}" + writer = RecordWriter(record_adapter_path) + nr_records = 1337 + + for record in generate_plain_records(nr_records): + writer.write(record) + writer.flush() + + nr_received_records = 0 + reader = RecordReader(record_adapter_path) + for _ in reader: + nr_received_records += 1 + + assert nr_records == nr_received_records + + +def test_avro_adapter_contextmanager(tmpdir: Path) -> None: + json_file = tmpdir.join("records.avro") + record_adapter_path = f"avro://{json_file}" + with RecordWriter(record_adapter_path) as writer: + nr_records = 1337 + for record in generate_plain_records(nr_records): + writer.write(record) + + nr_received_records = 0 + with RecordReader(record_adapter_path) as reader: + for _ in reader: + nr_received_records += 1 + + assert nr_records == nr_received_records + + +def test_avro_adapter_empty(tmpdir: Path) -> None: + json_file = tmpdir.join("records.avro") + record_adapter_path = f"avro://{json_file}" + with RecordWriter(record_adapter_path): + pass + + nr_received_records = 0 + with RecordReader(record_adapter_path) as reader: + for _ in reader: + nr_received_records += 1 + + assert nr_received_records == 0 diff --git a/tests/test_csv_adapter.py b/tests/adapters/test_csv.py similarity index 100% rename from tests/test_csv_adapter.py rename to tests/adapters/test_csv.py diff --git a/tests/test_elastic_adapter.py b/tests/adapters/test_elastic.py similarity index 100% rename from tests/test_elastic_adapter.py rename to tests/adapters/test_elastic.py diff --git a/tests/test_json_record_adapter.py b/tests/adapters/test_json.py similarity index 98% rename from tests/test_json_record_adapter.py rename to tests/adapters/test_json.py index 7aefd6f2..1e741d8a 100644 --- a/tests/test_json_record_adapter.py +++ b/tests/adapters/test_json.py @@ -6,8 +6,7 @@ import pytest from flow.record import RecordReader, RecordWriter - -from ._utils import generate_records +from tests._utils import generate_records if TYPE_CHECKING: from pathlib import Path diff --git a/tests/test_adapter_line.py b/tests/adapters/test_line.py similarity index 100% rename from tests/test_adapter_line.py rename to tests/adapters/test_line.py diff --git a/tests/test_splunk_adapter.py b/tests/adapters/test_splunk.py similarity index 100% rename from tests/test_splunk_adapter.py rename to tests/adapters/test_splunk.py diff --git a/tests/test_sqlite_duckdb_adapter.py b/tests/adapters/test_sqlite_duckdb.py similarity index 100% rename from tests/test_sqlite_duckdb_adapter.py rename to tests/adapters/test_sqlite_duckdb.py diff --git a/tests/test_adapter_text.py b/tests/adapters/test_text.py similarity index 100% rename from tests/test_adapter_text.py rename to tests/adapters/test_text.py diff --git a/tests/test_xlsx_adapter.py b/tests/adapters/test_xlsx.py similarity index 100% rename from tests/test_xlsx_adapter.py rename to tests/adapters/test_xlsx.py diff --git a/tests/fieldtypes/__init__.py b/tests/fieldtypes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_fieldtypes.py b/tests/fieldtypes/test_fieldtypes.py similarity index 100% rename from tests/test_fieldtypes.py rename to tests/fieldtypes/test_fieldtypes.py diff --git a/tests/test_fieldtype_ip.py b/tests/fieldtypes/test_ip.py similarity index 100% rename from tests/test_fieldtype_ip.py rename to tests/fieldtypes/test_ip.py diff --git a/tests/packers/__init__.py b/tests/packers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_json_packer.py b/tests/packers/test_json_packer.py similarity index 100% rename from tests/test_json_packer.py rename to tests/packers/test_json_packer.py diff --git a/tests/test_packer.py b/tests/packers/test_packer.py similarity index 100% rename from tests/test_packer.py rename to tests/packers/test_packer.py diff --git a/tests/records/__init__.py b/tests/records/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_record_adapter.py b/tests/records/test_adapter.py similarity index 99% rename from tests/test_record_adapter.py rename to tests/records/test_adapter.py index 8f821d73..b6cb8b37 100644 --- a/tests/test_record_adapter.py +++ b/tests/records/test_adapter.py @@ -28,8 +28,7 @@ ZSTD_MAGIC, ) from flow.record.selector import CompiledSelector, Selector - -from ._utils import generate_records +from tests._utils import generate_records if TYPE_CHECKING: from pathlib import Path diff --git a/tests/test_record_descriptor.py b/tests/records/test_descriptor.py similarity index 100% rename from tests/test_record_descriptor.py rename to tests/records/test_descriptor.py diff --git a/tests/test_multi_timestamp.py b/tests/records/test_multi_timestamp.py similarity index 100% rename from tests/test_multi_timestamp.py rename to tests/records/test_multi_timestamp.py diff --git a/tests/test_record.py b/tests/records/test_record.py similarity index 100% rename from tests/test_record.py rename to tests/records/test_record.py diff --git a/tests/selectors/__init__.py b/tests/selectors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_compiled_selector.py b/tests/selectors/test_compiled.py similarity index 100% rename from tests/test_compiled_selector.py rename to tests/selectors/test_compiled.py diff --git a/tests/test_selector.py b/tests/selectors/test_selectors.py similarity index 100% rename from tests/test_selector.py rename to tests/selectors/test_selectors.py diff --git a/tests/standalone_test.py b/tests/standalone_test.py deleted file mode 100644 index 6389754e..00000000 --- a/tests/standalone_test.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import annotations - -import traceback -from typing import Callable - - -def main(glob: dict[str, Callable[..., None]]) -> None: - for var, val in sorted(glob.items()): - if not var.startswith("test_"): - continue - - print(f"{var:40s}", end="") - try: - val() - print("PASSED") - except Exception: - print("FAILED") - - traceback.print_exc() diff --git a/tests/test_avro_adapter.py b/tests/test_avro_adapter.py deleted file mode 100644 index b1d8e10c..00000000 --- a/tests/test_avro_adapter.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from flow.record import RecordReader, RecordWriter - -from ._utils import generate_plain_records - -if TYPE_CHECKING: - from pathlib import Path - - -def test_avro_adapter(tmpdir: Path) -> None: - json_file = tmpdir.join("records.avro") - record_adapter_path = f"avro://{json_file}" - writer = RecordWriter(record_adapter_path) - nr_records = 1337 - - for record in generate_plain_records(nr_records): - writer.write(record) - writer.flush() - - nr_received_records = 0 - reader = RecordReader(record_adapter_path) - for _ in reader: - nr_received_records += 1 - - assert nr_records == nr_received_records - - -def test_avro_adapter_contextmanager(tmpdir: Path) -> None: - json_file = tmpdir.join("records.avro") - record_adapter_path = f"avro://{json_file}" - with RecordWriter(record_adapter_path) as writer: - nr_records = 1337 - for record in generate_plain_records(nr_records): - writer.write(record) - - nr_received_records = 0 - with RecordReader(record_adapter_path) as reader: - for _ in reader: - nr_received_records += 1 - - assert nr_records == nr_received_records - - -def test_avro_adapter_empty(tmpdir: Path) -> None: - json_file = tmpdir.join("records.avro") - record_adapter_path = f"avro://{json_file}" - with RecordWriter(record_adapter_path): - pass - - nr_received_records = 0 - with RecordReader(record_adapter_path) as reader: - for _ in reader: - nr_received_records += 1 - - assert nr_received_records == 0 diff --git a/tests/test_regression.py b/tests/test_regressions.py similarity index 100% rename from tests/test_regression.py rename to tests/test_regressions.py diff --git a/tests/tools/__init__.py b/tests/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_rdump.py b/tests/tools/test_rdump.py similarity index 100% rename from tests/test_rdump.py rename to tests/tools/test_rdump.py diff --git a/tox.ini b/tox.ini index 0e0150df..90fa2c67 100644 --- a/tox.ini +++ b/tox.ini @@ -57,12 +57,12 @@ deps = sphinx-design furo commands = - make -C tests/docs clean - make -C tests/docs html + make -C tests/_docs clean + make -C tests/_docs html [testenv:docs-linkcheck] allowlist_externals = make deps = {[testenv:docs-build]deps} commands = - make -C tests/docs clean - make -C tests/docs linkcheck + make -C tests/_docs clean + make -C tests/_docs linkcheck From 85cf70de8f0dbf3e01aa5cb012c854e2c2132fa6 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 6 Aug 2025 11:20:07 +0200 Subject: [PATCH 2/2] rename folders --- tests/{adapters => adapter}/__init__.py | 0 tests/{adapters => adapter}/test_avro.py | 0 tests/{adapters => adapter}/test_csv.py | 0 tests/{adapters => adapter}/test_elastic.py | 0 tests/{adapters => adapter}/test_json.py | 0 tests/{adapters => adapter}/test_line.py | 0 tests/{adapters => adapter}/test_splunk.py | 0 tests/{adapters => adapter}/test_sqlite_duckdb.py | 0 tests/{adapters => adapter}/test_text.py | 0 tests/{adapters => adapter}/test_xlsx.py | 0 tests/{packers => packer}/__init__.py | 0 tests/{packers => packer}/test_json_packer.py | 0 tests/{packers => packer}/test_packer.py | 0 tests/{records => record}/__init__.py | 0 tests/{records => record}/test_adapter.py | 0 tests/{records => record}/test_descriptor.py | 0 tests/{records => record}/test_multi_timestamp.py | 0 tests/{records => record}/test_record.py | 0 tests/{selectors => selector}/__init__.py | 0 tests/{selectors => selector}/test_compiled.py | 0 tests/{selectors => selector}/test_selectors.py | 0 21 files changed, 0 insertions(+), 0 deletions(-) rename tests/{adapters => adapter}/__init__.py (100%) rename tests/{adapters => adapter}/test_avro.py (100%) rename tests/{adapters => adapter}/test_csv.py (100%) rename tests/{adapters => adapter}/test_elastic.py (100%) rename tests/{adapters => adapter}/test_json.py (100%) rename tests/{adapters => adapter}/test_line.py (100%) rename tests/{adapters => adapter}/test_splunk.py (100%) rename tests/{adapters => adapter}/test_sqlite_duckdb.py (100%) rename tests/{adapters => adapter}/test_text.py (100%) rename tests/{adapters => adapter}/test_xlsx.py (100%) rename tests/{packers => packer}/__init__.py (100%) rename tests/{packers => packer}/test_json_packer.py (100%) rename tests/{packers => packer}/test_packer.py (100%) rename tests/{records => record}/__init__.py (100%) rename tests/{records => record}/test_adapter.py (100%) rename tests/{records => record}/test_descriptor.py (100%) rename tests/{records => record}/test_multi_timestamp.py (100%) rename tests/{records => record}/test_record.py (100%) rename tests/{selectors => selector}/__init__.py (100%) rename tests/{selectors => selector}/test_compiled.py (100%) rename tests/{selectors => selector}/test_selectors.py (100%) diff --git a/tests/adapters/__init__.py b/tests/adapter/__init__.py similarity index 100% rename from tests/adapters/__init__.py rename to tests/adapter/__init__.py diff --git a/tests/adapters/test_avro.py b/tests/adapter/test_avro.py similarity index 100% rename from tests/adapters/test_avro.py rename to tests/adapter/test_avro.py diff --git a/tests/adapters/test_csv.py b/tests/adapter/test_csv.py similarity index 100% rename from tests/adapters/test_csv.py rename to tests/adapter/test_csv.py diff --git a/tests/adapters/test_elastic.py b/tests/adapter/test_elastic.py similarity index 100% rename from tests/adapters/test_elastic.py rename to tests/adapter/test_elastic.py diff --git a/tests/adapters/test_json.py b/tests/adapter/test_json.py similarity index 100% rename from tests/adapters/test_json.py rename to tests/adapter/test_json.py diff --git a/tests/adapters/test_line.py b/tests/adapter/test_line.py similarity index 100% rename from tests/adapters/test_line.py rename to tests/adapter/test_line.py diff --git a/tests/adapters/test_splunk.py b/tests/adapter/test_splunk.py similarity index 100% rename from tests/adapters/test_splunk.py rename to tests/adapter/test_splunk.py diff --git a/tests/adapters/test_sqlite_duckdb.py b/tests/adapter/test_sqlite_duckdb.py similarity index 100% rename from tests/adapters/test_sqlite_duckdb.py rename to tests/adapter/test_sqlite_duckdb.py diff --git a/tests/adapters/test_text.py b/tests/adapter/test_text.py similarity index 100% rename from tests/adapters/test_text.py rename to tests/adapter/test_text.py diff --git a/tests/adapters/test_xlsx.py b/tests/adapter/test_xlsx.py similarity index 100% rename from tests/adapters/test_xlsx.py rename to tests/adapter/test_xlsx.py diff --git a/tests/packers/__init__.py b/tests/packer/__init__.py similarity index 100% rename from tests/packers/__init__.py rename to tests/packer/__init__.py diff --git a/tests/packers/test_json_packer.py b/tests/packer/test_json_packer.py similarity index 100% rename from tests/packers/test_json_packer.py rename to tests/packer/test_json_packer.py diff --git a/tests/packers/test_packer.py b/tests/packer/test_packer.py similarity index 100% rename from tests/packers/test_packer.py rename to tests/packer/test_packer.py diff --git a/tests/records/__init__.py b/tests/record/__init__.py similarity index 100% rename from tests/records/__init__.py rename to tests/record/__init__.py diff --git a/tests/records/test_adapter.py b/tests/record/test_adapter.py similarity index 100% rename from tests/records/test_adapter.py rename to tests/record/test_adapter.py diff --git a/tests/records/test_descriptor.py b/tests/record/test_descriptor.py similarity index 100% rename from tests/records/test_descriptor.py rename to tests/record/test_descriptor.py diff --git a/tests/records/test_multi_timestamp.py b/tests/record/test_multi_timestamp.py similarity index 100% rename from tests/records/test_multi_timestamp.py rename to tests/record/test_multi_timestamp.py diff --git a/tests/records/test_record.py b/tests/record/test_record.py similarity index 100% rename from tests/records/test_record.py rename to tests/record/test_record.py diff --git a/tests/selectors/__init__.py b/tests/selector/__init__.py similarity index 100% rename from tests/selectors/__init__.py rename to tests/selector/__init__.py diff --git a/tests/selectors/test_compiled.py b/tests/selector/test_compiled.py similarity index 100% rename from tests/selectors/test_compiled.py rename to tests/selector/test_compiled.py diff --git a/tests/selectors/test_selectors.py b/tests/selector/test_selectors.py similarity index 100% rename from tests/selectors/test_selectors.py rename to tests/selector/test_selectors.py