From 86c4bcc4ad2906cd6883635a92adc9004191a936 Mon Sep 17 00:00:00 2001 From: Miauwkeru Date: Wed, 11 Mar 2026 12:36:44 +0000 Subject: [PATCH 1/8] Add fold to the datetime fieldtype This is for disambiguation during dst time --- flow/record/fieldtypes/__init__.py | 1 + tests/fieldtypes/test_fieldtypes.py | 37 ++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/flow/record/fieldtypes/__init__.py b/flow/record/fieldtypes/__init__.py index 5aa97ad5..816cf72c 100644 --- a/flow/record/fieldtypes/__init__.py +++ b/flow/record/fieldtypes/__init__.py @@ -308,6 +308,7 @@ def __new__(cls, *args, **kwargs): arg.second, arg.microsecond, tzinfo, + fold=arg.fold, ) else: obj = _dt.__new__(cls, *args, **kwargs) diff --git a/tests/fieldtypes/test_fieldtypes.py b/tests/fieldtypes/test_fieldtypes.py index 65e2b1fd..481a8765 100644 --- a/tests/fieldtypes/test_fieldtypes.py +++ b/tests/fieldtypes/test_fieldtypes.py @@ -21,6 +21,7 @@ _is_windowslike_path, command, fieldtype_for_value, + flow_record_tz, net, posix_command, posix_path, @@ -459,7 +460,7 @@ def test_datetime() -> None: ("2006-11-10T14:29:55.585192699999999-07:00", datetime(2006, 11, 10, 21, 29, 55, 585192, tzinfo=UTC)), ], ) -def test_datetime_formats(tmp_path: pathlib.Path, value: str, expected_dt: datetime) -> None: +def test_datetime_formats(tmp_path: pathlib.Path, value: str | datetime, expected_dt: datetime) -> None: TestRecord = RecordDescriptor( "test/datetime", [ @@ -480,6 +481,40 @@ def test_datetime_formats(tmp_path: pathlib.Path, value: str, expected_dt: datet assert record.dt == expected_dt +@pytest.mark.parametrize( + ("value", "expected_dt"), + [ + (datetime(2023, 1, 1, tzinfo=UTC, fold=1), datetime(2023, 1, 1, tzinfo=UTC)), + ( # "W. Europe Standard Time" + datetime(2025, 10, 26, 2, 0, 3, tzinfo=flow_record_tz(default_tz="Europe/Amsterdam"), fold=1), + datetime(2025, 10, 26, 1, 0, 3, tzinfo=UTC), + ), + ], +) +def test_datetime_formats_fold(tmp_path: pathlib.Path, value: datetime, expected_dt: datetime) -> None: + """test whether datetime accepts fold parameters and converts it correctly""" + TestRecord = RecordDescriptor( + "test/datetime", + [ + ("datetime", "dt"), + ], + ) + record = TestRecord(dt=value) + assert record.dt.fold == 1 + assert record.dt.astimezone(UTC) == expected_dt + + # test packing / serialization of datetime fields + path = tmp_path / "datetime.records" + with RecordWriter(path) as writer: + writer.write(record) + + # test unpacking / deserialization of datetime fields + with RecordReader(path) as reader: + record = next(iter(reader)) + # Need to convert it to UTC specifically as the timezones do not match + assert record.dt.astimezone(UTC) == expected_dt + + def test_digest() -> None: TestRecord = RecordDescriptor( "test/digest", From f88478e0b844ae1e120289e5e14ff3a2a8767aee Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Wed, 11 Mar 2026 21:48:30 +0100 Subject: [PATCH 2/8] Use ZoneInfo over flow_record_tz --- tests/fieldtypes/test_fieldtypes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fieldtypes/test_fieldtypes.py b/tests/fieldtypes/test_fieldtypes.py index 94b840b9..796dc664 100644 --- a/tests/fieldtypes/test_fieldtypes.py +++ b/tests/fieldtypes/test_fieldtypes.py @@ -17,11 +17,11 @@ PY_313_OR_HIGHER, TYPE_POSIX, TYPE_WINDOWS, + ZoneInfo, _is_posixlike_path, _is_windowslike_path, command, fieldtype_for_value, - flow_record_tz, net, posix_path, uri, @@ -453,8 +453,8 @@ def test_datetime_formats(tmp_path: pathlib.Path, value: str | datetime, expecte ("value", "expected_dt"), [ (datetime(2023, 1, 1, tzinfo=UTC, fold=1), datetime(2023, 1, 1, tzinfo=UTC)), - ( # "W. Europe Standard Time" - datetime(2025, 10, 26, 2, 0, 3, tzinfo=flow_record_tz(default_tz="Europe/Amsterdam"), fold=1), + ( + datetime(2025, 10, 26, 2, 0, 3, tzinfo=ZoneInfo("Europe/Amsterdam"), fold=1), datetime(2025, 10, 26, 1, 0, 3, tzinfo=UTC), ), ], From d47f174752e72cdfa00a26f35f787369367f4497 Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Wed, 11 Mar 2026 21:48:47 +0100 Subject: [PATCH 3/8] Added more written out test case involving datetime fold --- tests/fieldtypes/test_fieldtypes.py | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/fieldtypes/test_fieldtypes.py b/tests/fieldtypes/test_fieldtypes.py index 796dc664..c14eff18 100644 --- a/tests/fieldtypes/test_fieldtypes.py +++ b/tests/fieldtypes/test_fieldtypes.py @@ -483,6 +483,43 @@ def test_datetime_formats_fold(tmp_path: pathlib.Path, value: datetime, expected assert record.dt.astimezone(UTC) == expected_dt +def test_datetime_fold_example() -> None: + """ + Test datetime fold parameter during daylight saving time changes in the Netherlands. + which has a timezone offset of +01:00 during wintertime and +02:00 during summertime. + """ + + TestRecord = RecordDescriptor( + "test/datetime", + [ + ("datetime", "dt"), + ], + ) + + # 2025-10-26 is the date of the end of daylight saving time in the Netherlands. + # At 3:00 AM the clock goes back to 2:00 AM, so 2:00 AM occurs twice. The first occurrence has fold=0 + record = TestRecord("2025-10-26T00:00:00+00:00") + nl_dt = record.dt.astimezone(ZoneInfo("Europe/Amsterdam")) + assert nl_dt.isoformat() == "2025-10-26T02:00:00+02:00" + assert nl_dt.hour == 2 + assert nl_dt.fold == 0 + + # test that both datetimes are considered equal when converted to UTC + record2 = TestRecord(nl_dt) + assert record.dt.astimezone(UTC) == record2.dt.astimezone(UTC) + + # wintertime, the clock goes back from 3:00 AM to 2:00 AM, so 2:00 AM occurs twice. The second occurrence has fold=1 + record = TestRecord("2025-10-26T01:00:00+00:00") + nl_dt = record.dt.astimezone(ZoneInfo("Europe/Amsterdam")) + assert nl_dt.isoformat() == "2025-10-26T02:00:00+01:00" + assert nl_dt.hour == 2 + assert nl_dt.fold == 1 + + # test that both datetimes are considered equal when converted to UTC + record2 = TestRecord(nl_dt) + assert record.dt.astimezone(UTC) == record2.dt.astimezone(UTC) + + def test_digest() -> None: TestRecord = RecordDescriptor( "test/digest", From 53a4f9868258e57fefcdf30308bc687c17601090 Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Wed, 11 Mar 2026 22:11:53 +0100 Subject: [PATCH 4/8] Update tests/fieldtypes/test_fieldtypes.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/fieldtypes/test_fieldtypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fieldtypes/test_fieldtypes.py b/tests/fieldtypes/test_fieldtypes.py index c14eff18..564b90b2 100644 --- a/tests/fieldtypes/test_fieldtypes.py +++ b/tests/fieldtypes/test_fieldtypes.py @@ -485,8 +485,8 @@ def test_datetime_formats_fold(tmp_path: pathlib.Path, value: datetime, expected def test_datetime_fold_example() -> None: """ - Test datetime fold parameter during daylight saving time changes in the Netherlands. - which has a timezone offset of +01:00 during wintertime and +02:00 during summertime. + Test datetime fold parameter during daylight saving time changes in the Netherlands, which has a + timezone offset of +01:00 during wintertime and +02:00 during summertime. """ TestRecord = RecordDescriptor( From 64ad910e58badd5935436d7ca3624c78ecb0feab Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Wed, 11 Mar 2026 22:17:11 +0100 Subject: [PATCH 5/8] Only run tests if HAS_ZONE_INFO due to ZoneInfo dependency --- tests/fieldtypes/test_fieldtypes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/fieldtypes/test_fieldtypes.py b/tests/fieldtypes/test_fieldtypes.py index 564b90b2..c68ecedc 100644 --- a/tests/fieldtypes/test_fieldtypes.py +++ b/tests/fieldtypes/test_fieldtypes.py @@ -13,11 +13,11 @@ import flow.record.fieldtypes from flow.record import RecordDescriptor, RecordReader, RecordWriter, fieldtypes from flow.record.fieldtypes import ( + HAS_ZONE_INFO, PY_312_OR_HIGHER, PY_313_OR_HIGHER, TYPE_POSIX, TYPE_WINDOWS, - ZoneInfo, _is_posixlike_path, _is_windowslike_path, command, @@ -29,6 +29,9 @@ ) from flow.record.fieldtypes import datetime as dt +if HAS_ZONE_INFO: + from flow.record.fieldtypes import ZoneInfo + if TYPE_CHECKING: from collections.abc import Callable @@ -449,6 +452,7 @@ def test_datetime_formats(tmp_path: pathlib.Path, value: str | datetime, expecte assert record.dt == expected_dt +@pytest.mark.skipifnot(HAS_ZONE_INFO, reason="ZoneInfo is required for testing datetime fold parameter") @pytest.mark.parametrize( ("value", "expected_dt"), [ @@ -483,6 +487,7 @@ def test_datetime_formats_fold(tmp_path: pathlib.Path, value: datetime, expected assert record.dt.astimezone(UTC) == expected_dt +@pytest.mark.skipifnot(HAS_ZONE_INFO, reason="ZoneInfo is required for testing datetime fold parameter") def test_datetime_fold_example() -> None: """ Test datetime fold parameter during daylight saving time changes in the Netherlands, which has a From 01a6d01dc8c02f7568b800751b90918eb820f092 Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Wed, 11 Mar 2026 22:35:15 +0100 Subject: [PATCH 6/8] Better pytest.skip --- tests/fieldtypes/test_fieldtypes.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/fieldtypes/test_fieldtypes.py b/tests/fieldtypes/test_fieldtypes.py index c68ecedc..494e7083 100644 --- a/tests/fieldtypes/test_fieldtypes.py +++ b/tests/fieldtypes/test_fieldtypes.py @@ -452,17 +452,20 @@ def test_datetime_formats(tmp_path: pathlib.Path, value: str | datetime, expecte assert record.dt == expected_dt -@pytest.mark.skipifnot(HAS_ZONE_INFO, reason="ZoneInfo is required for testing datetime fold parameter") -@pytest.mark.parametrize( - ("value", "expected_dt"), - [ - (datetime(2023, 1, 1, tzinfo=UTC, fold=1), datetime(2023, 1, 1, tzinfo=UTC)), +DATETIME_FOLD_PARAMS = [ + (datetime(2023, 1, 1, tzinfo=UTC, fold=1), datetime(2023, 1, 1, tzinfo=UTC)), +] +if HAS_ZONE_INFO: + DATETIME_FOLD_PARAMS.append( ( datetime(2025, 10, 26, 2, 0, 3, tzinfo=ZoneInfo("Europe/Amsterdam"), fold=1), datetime(2025, 10, 26, 1, 0, 3, tzinfo=UTC), ), - ], -) + ) + + +@pytest.mark.skipifnot(HAS_ZONE_INFO, reason="ZoneInfo is required for testing datetime fold parameter") +@pytest.mark.parametrize(("value", "expected_dt"), DATETIME_FOLD_PARAMS) def test_datetime_formats_fold(tmp_path: pathlib.Path, value: datetime, expected_dt: datetime) -> None: """test whether datetime accepts fold parameters and converts it correctly""" TestRecord = RecordDescriptor( From 3319c741600c5e5f5139cdb54ab806cdb18191ff Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Wed, 11 Mar 2026 22:36:57 +0100 Subject: [PATCH 7/8] Improve type annotation --- tests/fieldtypes/test_fieldtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fieldtypes/test_fieldtypes.py b/tests/fieldtypes/test_fieldtypes.py index 494e7083..0662ae75 100644 --- a/tests/fieldtypes/test_fieldtypes.py +++ b/tests/fieldtypes/test_fieldtypes.py @@ -431,7 +431,7 @@ def test_datetime() -> None: ("2006-11-10T14:29:55.585192699999999-07:00", datetime(2006, 11, 10, 21, 29, 55, 585192, tzinfo=UTC)), ], ) -def test_datetime_formats(tmp_path: pathlib.Path, value: str | datetime, expected_dt: datetime) -> None: +def test_datetime_formats(tmp_path: pathlib.Path, value: str | datetime | float, expected_dt: datetime) -> None: TestRecord = RecordDescriptor( "test/datetime", [ From 4925eb10fe1a48bdb11f25ed65d8d20c7cc3e6fe Mon Sep 17 00:00:00 2001 From: Yun Zheng Hu Date: Wed, 11 Mar 2026 23:05:00 +0100 Subject: [PATCH 8/8] Fix pytest.mark.skipif --- tests/fieldtypes/test_fieldtypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fieldtypes/test_fieldtypes.py b/tests/fieldtypes/test_fieldtypes.py index 0662ae75..08a273f7 100644 --- a/tests/fieldtypes/test_fieldtypes.py +++ b/tests/fieldtypes/test_fieldtypes.py @@ -464,7 +464,7 @@ def test_datetime_formats(tmp_path: pathlib.Path, value: str | datetime | float, ) -@pytest.mark.skipifnot(HAS_ZONE_INFO, reason="ZoneInfo is required for testing datetime fold parameter") +@pytest.mark.skipif(not HAS_ZONE_INFO, reason="ZoneInfo is required for testing datetime fold parameter") @pytest.mark.parametrize(("value", "expected_dt"), DATETIME_FOLD_PARAMS) def test_datetime_formats_fold(tmp_path: pathlib.Path, value: datetime, expected_dt: datetime) -> None: """test whether datetime accepts fold parameters and converts it correctly""" @@ -490,7 +490,7 @@ def test_datetime_formats_fold(tmp_path: pathlib.Path, value: datetime, expected assert record.dt.astimezone(UTC) == expected_dt -@pytest.mark.skipifnot(HAS_ZONE_INFO, reason="ZoneInfo is required for testing datetime fold parameter") +@pytest.mark.skipif(not HAS_ZONE_INFO, reason="ZoneInfo is required for testing datetime fold parameter") def test_datetime_fold_example() -> None: """ Test datetime fold parameter during daylight saving time changes in the Netherlands, which has a