2396 check that all entity numbers in lookupcsv are contained within the ranges in entity organisationcsv

bin/fix_dates.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+import csv
+import re
+from pathlib import Path
+
+TARGET_DIRS = ("pipeline", "specification", "collection")
+
+# Matches malformed form: 2023-06-23T10:10:11:49Z
+BAD_TS_RE = re.compile(
+    r"^(\d{4}-\d{2}-\d{2})T(\d{2}):(\d{2}):(\d{2}):(\d{1,6})Z$"
+)
+
+def normalize_timestamp(value: str) -> str:
+    if not isinstance(value, str):
+        return value
+    s = value.strip()
+    m = BAD_TS_RE.match(s)
+    if not m:
+        return value
+
+    date_part, hh, mm, ss, frac = m.groups()
+    # Convert trailing part to milliseconds (pad/truncate to 3 digits)
+    ms = frac.ljust(3, "0")[:3]
+    return f"{date_part}T{hh}:{mm}:{ss}.{ms}Z"
+
+def fix_csv_file(csv_path: Path) -> int:
+    changed = 0
+    with csv_path.open("r", newline="", encoding="utf-8") as f:
+        reader = csv.DictReader(f)
+        rows = list(reader)
+        fieldnames = reader.fieldnames or []
+
+    if not rows:
+        return 0
+
+    for row in rows:
+        for k, v in row.items():
+            new_v = normalize_timestamp(v)
+            if new_v != v:
+                row[k] = new_v
+                changed += 1
+
+    if changed:
+        with csv_path.open("w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(rows)
+
+    return changed
+
+def main(root="."):
+    root_path = Path(root)
+    total_files_changed = 0
+    total_values_changed = 0
+
+    for target_dir in TARGET_DIRS:
+        base_dir = root_path / target_dir
+        if not base_dir.exists() or not base_dir.is_dir():
+            continue
+
+        for csv_file in base_dir.rglob("*.csv"):
+            changes = fix_csv_file(csv_file)
+            if changes:
+                total_files_changed += 1
+                total_values_changed += changes
+                print(f"Updated {csv_file} ({changes} value(s))")
+
+    print(
+        f"Done. Files changed: {total_files_changed}, "
+        f"timestamps fixed: {total_values_changed}"
+    )
+
+if __name__ == "__main__":
+    main(".")

collection/area-of-outstanding-natural-beauty/endpoint.csv

@@ -1,4 +1,4 @@
 endpoint,endpoint-url,parameters,plugin,entry-date,start-date,end-date
 4ecea72eff7de59214fa5919fa011081919d523601656fabd053e2a5e19fa96d,https://opendata.arcgis.com/datasets/6f2ad07d91304ad79cdecd52489d5046_0.geojson,,,2021-05-25T11:45:20Z,2021-05-25,2024-10-01
-34167c688ca765477c47e63592ff9149bb46710a0850f6c9b61b5aac25f09823,https://raw.githubusercontent.com/digital-land/area-of-outstanding-natural-beauty-collection/main/data/area-of-outstanding-natural-beauty.csv,,,2022-09-10T16:16:46:19Z,,
+34167c688ca765477c47e63592ff9149bb46710a0850f6c9b61b5aac25f09823,https://raw.githubusercontent.com/digital-land/area-of-outstanding-natural-beauty-collection/main/data/area-of-outstanding-natural-beauty.csv,,,2022-09-10T16:16:46.190Z,,
 72dbf89a640ebc84604afc5dc9b1f1e0f498ad01faf5f15baabe42af9ac365e0,https://services.arcgis.com/JJzESW51TqeY9uat/arcgis/rest/services/Areas_of_Outstanding_Natural_Beauty_England/FeatureServer/0/query?where=1%3D1&outfields=*&f=geojson,,,2024-10-01,2024-10-01,

collection/area-of-outstanding-natural-beauty/source.csv

@@ -1,4 +1,4 @@
 source,attribution,collection,documentation-url,endpoint,licence,organisation,pipelines,entry-date,start-date,end-date
 389de95c36ece9177ef97afb66cc8ed3,natural-england,area-of-outstanding-natural-beauty,https://naturalengland-defra.opendata.arcgis.com/datasets/areas-of-outstanding-natural-beauty-england,4ecea72eff7de59214fa5919fa011081919d523601656fabd053e2a5e19fa96d,ogl3,government-organisation:PB202,area-of-outstanding-natural-beauty,2021-05-25T00:00:00Z,2020-09-01,2024-10-01
-29b6cd2da1d6dc15842ed4390e1dd7ef,crown-copyright,area-of-outstanding-natural-beauty,,34167c688ca765477c47e63592ff9149bb46710a0850f6c9b61b5aac25f09823,ogl3,government-organisation:D1342,area-of-outstanding-natural-beauty,2022-09-10T16:16:46:19Z,,
+29b6cd2da1d6dc15842ed4390e1dd7ef,crown-copyright,area-of-outstanding-natural-beauty,,34167c688ca765477c47e63592ff9149bb46710a0850f6c9b61b5aac25f09823,ogl3,government-organisation:D1342,area-of-outstanding-natural-beauty,2022-09-10T16:16:46.190Z,,
 b64abf8921effb9805599db5aeeb821f,natural-england,area-of-outstanding-natural-beauty,https://naturalengland-defra.opendata.arcgis.com/datasets/Defra::areas-of-outstanding-natural-beauty-england/about,72dbf89a640ebc84604afc5dc9b1f1e0f498ad01faf5f15baabe42af9ac365e0,ogl3,government-organisation:PB202,area-of-outstanding-natural-beauty,2024-10-01,2024-10-01,

collection/article-4-direction/endpoint.csv

@@ -15,33 +15,33 @@ d107d44c64f3987dac0ac9c4907553d522772960aad4ea1b355e903ba252c094,https://raw.git
 0c73554d24de07895c68bcbc38e958a9776348338ff2493dae5a22fe584c24d1,https://raw.githubusercontent.com/digital-land/article-4-direction-collection/main/data/Southwark/Article%204%20_%20Public%20Houses.gpkg,,,2022-04-06T16:59:34Z,,2024-02-12
 f3a8c9747bd876f26208ddfe0dddcf2fd6354ec88fa6eb671575187781bbb783,https://raw.githubusercontent.com/digital-land/article-4-direction-collection/main/data/Southwark/Article%204%20_%20Railway%20Arches.gpkg,,,2022-04-06T17:00:00Z,,2024-02-12
 8a6b78daac84a927753288627605f45f20ecdfc27ed8c0bd9a2b739be184af1b,https://raw.githubusercontent.com/digital-land/article-4-direction-collection/main/data/Southwark/Article%204%20_%20Sunray%20Estate.gpkg,,,2022-04-06T17:00:26Z,,2024-02-12
-7b50969a85c19aec02a07248a5aabde97ea0fe835ca5e6b5e6b02dffb7a8b8ef,https://raw.githubusercontent.com/digital-land/article-4-direction-collection/main/data/Southwark/article-4-direction.csv,,,2022-04-25T19:19:59:10Z,,2024-05-21
-7611c4b234779a4a7783655b0f06f0d72f0a2b2022eec2319b4e00c7f7d90e4c,https://maps.buckscc.gov.uk/arcgis/services/PLANNING/A4D_AV/MapServer/WFSServer?request=GetFeature&service=WFS&typename=PLANNING_A4D_AV:Article_4_Direction_s__Aylesbury_Vale_,,,2022-04-28T15:15:08:21Z,,2024-10-01
-c894c55d36eee21406016aa0424b4b6e398e289169f0394b6c8d9c81b325c2a4,https://maps.buckscc.gov.uk/arcgis/services/PLANNING/A4D_CDC/MapServer/WFSServer?request=GetFeature&service=WFS&typename=PLANNING_A4D_CDC:Article_4_Direction_s__Chiltern_,,,2022-04-28T16:16:51:48Z,,2024-09-05
-f949534bb3d2410aa82d31f1f2f3e8a12aa9c4fc06ec7bb9b385a9b9e97caf0f,https://maps.buckscc.gov.uk/arcgis/services/PLANNING/A4D_SBDC/MapServer/WFSServer?request=GetFeature&service=WFS&typename=PLANNING_A4D_SBDC:Article_4_Direction_s__South_Bucks_,,,2022-04-28T16:16:56:34Z,,2024-09-05
-49cbcdf8780dc91403d01ad916b118d31da3d8996f3354f6ef4ea06994dbaa46,https://maps.buckscc.gov.uk/arcgis/services/PLANNING/A4D_WDC/MapServer/WFSServer?request=GetFeature&service=WFS&typename=PLANNING_A4D_WDC:Article_4_Direction_s__Wycombe_,,wfs,2022-04-28T16:16:57:58Z,,2024-10-10
-5579906cbdd8b0c2039628f8574b9d94df510bee323811351bb25ea2bc7c34ef,https://raw.githubusercontent.com/digital-land/article-4-direction-collection/main/data/Buckinghamshire/patches.csv,,,2022-05-12T09:09:50:56Z,,
-d779ad1c91c5a46e2d4ace4d5446d7d7f81df1ed058f882121070574697a5412,http://medwaymaps.medway.gov.uk/WFS/wfs.exe?service=WFS&version=2.0.0&request=GetFeature&TypeNames=DVL_ARTICLE_4,,wfs,2022-06-30T07:07:44:31Z,,
-5fc20cc8e733d12f262bab292b81cb38ff35fb8d17b8503005403652759ba7b3,https://gcty.dynamicmaps.co.uk:8443/geoserver/Digital_Land/ows?service=wfs&request=GetFeature&TypeNames=Digital_Land%3Aarticle_4_DL&OutputFormat=GML2,,wfs,2022-06-30T09:09:45:45Z,,2024-07-02
-83aa4b6b9a8e042c9ab842f91994835c94a9f2cfb6bb2b75908216fa77bde191,https://dservices2.arcgis.com/LrUbY6lLLgV3tEa5/arcgis/services/Article_4_Directions_View_WFS/WFSServer?service=wfs&request=GetFeature&TypeNames=Article_4&OutputFormat=GML2,,wfs,2022-06-30T09:09:56:24Z,,2024-10-23
-e5598a75328d5890efb7e6d8bb6c1d181fbed2cda45af24de6627f60e777bbe2,https://raw.githubusercontent.com/digital-land/article-4-direction-collection/main/data/article-4-direction-rule.csv,,,2022-10-11T17:17:20:33Z,,
+7b50969a85c19aec02a07248a5aabde97ea0fe835ca5e6b5e6b02dffb7a8b8ef,https://raw.githubusercontent.com/digital-land/article-4-direction-collection/main/data/Southwark/article-4-direction.csv,,,2022-04-25T19:19:59.100Z,,2024-05-21
+7611c4b234779a4a7783655b0f06f0d72f0a2b2022eec2319b4e00c7f7d90e4c,https://maps.buckscc.gov.uk/arcgis/services/PLANNING/A4D_AV/MapServer/WFSServer?request=GetFeature&service=WFS&typename=PLANNING_A4D_AV:Article_4_Direction_s__Aylesbury_Vale_,,,2022-04-28T15:15:08.210Z,,2024-10-01
+c894c55d36eee21406016aa0424b4b6e398e289169f0394b6c8d9c81b325c2a4,https://maps.buckscc.gov.uk/arcgis/services/PLANNING/A4D_CDC/MapServer/WFSServer?request=GetFeature&service=WFS&typename=PLANNING_A4D_CDC:Article_4_Direction_s__Chiltern_,,,2022-04-28T16:16:51.480Z,,2024-09-05
+f949534bb3d2410aa82d31f1f2f3e8a12aa9c4fc06ec7bb9b385a9b9e97caf0f,https://maps.buckscc.gov.uk/arcgis/services/PLANNING/A4D_SBDC/MapServer/WFSServer?request=GetFeature&service=WFS&typename=PLANNING_A4D_SBDC:Article_4_Direction_s__South_Bucks_,,,2022-04-28T16:16:56.340Z,,2024-09-05
+49cbcdf8780dc91403d01ad916b118d31da3d8996f3354f6ef4ea06994dbaa46,https://maps.buckscc.gov.uk/arcgis/services/PLANNING/A4D_WDC/MapServer/WFSServer?request=GetFeature&service=WFS&typename=PLANNING_A4D_WDC:Article_4_Direction_s__Wycombe_,,wfs,2022-04-28T16:16:57.580Z,,2024-10-10
+5579906cbdd8b0c2039628f8574b9d94df510bee323811351bb25ea2bc7c34ef,https://raw.githubusercontent.com/digital-land/article-4-direction-collection/main/data/Buckinghamshire/patches.csv,,,2022-05-12T09:09:50.560Z,,
+d779ad1c91c5a46e2d4ace4d5446d7d7f81df1ed058f882121070574697a5412,http://medwaymaps.medway.gov.uk/WFS/wfs.exe?service=WFS&version=2.0.0&request=GetFeature&TypeNames=DVL_ARTICLE_4,,wfs,2022-06-30T07:07:44.310Z,,
+5fc20cc8e733d12f262bab292b81cb38ff35fb8d17b8503005403652759ba7b3,https://gcty.dynamicmaps.co.uk:8443/geoserver/Digital_Land/ows?service=wfs&request=GetFeature&TypeNames=Digital_Land%3Aarticle_4_DL&OutputFormat=GML2,,wfs,2022-06-30T09:09:45.450Z,,2024-07-02
+83aa4b6b9a8e042c9ab842f91994835c94a9f2cfb6bb2b75908216fa77bde191,https://dservices2.arcgis.com/LrUbY6lLLgV3tEa5/arcgis/services/Article_4_Directions_View_WFS/WFSServer?service=wfs&request=GetFeature&TypeNames=Article_4&OutputFormat=GML2,,wfs,2022-06-30T09:09:56.240Z,,2024-10-23
+e5598a75328d5890efb7e6d8bb6c1d181fbed2cda45af24de6627f60e777bbe2,https://raw.githubusercontent.com/digital-land/article-4-direction-collection/main/data/article-4-direction-rule.csv,,,2022-10-11T17:17:20.330Z,,
 657ede08e75ebdacbf21522a8f312e05297cd19ece206c92512db2894947dee6,https://maps.doncaster.gov.uk/server/rest/services/Planning/Article_4/MapServer/0/,,arcgis,2022-11-07T13:57:24Z,,
-16857564d1658edeaa7703207b91525d005e7696e2e9392fbb6fa4231fa550ef,https://datamillnorth.org/download/newcastle-article-4-directions/4c650010-8d88-4bba-9f4a-85587742a64d/article4directionareas.csv,,,2023-02-23T22:22:12:39Z,,2024-06-19
-d788f7a3f32b918aca77f75952243261fae0f6eddc6d32636a8955198d0293fd,https://datamillnorth.org/download/newcastle-article-4-directions/5e84e19f-028c-4902-8a23-bd0dd3f388eb/article4directions.csv,,,2023-02-28T11:11:10:23Z,,2024-06-19
-3bbf7138bc9dacbf42700c5d114ba0c1856ae4cdc7c92360e95103ae59d38c0a,https://opendata.camden.gov.uk/api/views/45mg-zjup/rows.csv,,,2023-08-11T16:16:49:18Z,,
-fc6886730a30350775d915a11a8bf5644e42c32395315abd525665a02246751a,https://maps.doncaster.gov.uk/server/rest/services/Planning/Article_4_DSA/MapServer/0,,arcgis,2023-08-30T10:10:04:07Z,,2024-02-05
-22b68af5f133601d5bafc187546b06f0cfffb95a1a6166449cec471ac7ed6706,https://opendata.camden.gov.uk/api/views/ypia-t7j2/rows.csv,,,2023-09-05T10:10:05:41Z,,
-24bc10a266141363496a758b610670d8fbc48495d8cd2e84a362485f0b93e861,https://services7.arcgis.com/NXULfNGIlsKmpCJD/arcgis/rest/services/Article_4_Direction_Area/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson,,,2023-10-09T12:12:45:23Z,,
-d7e334900b5355ff4b43323d146d9d72cd3647bb4b31025aa7bd05dcb69596de,https://services7.arcgis.com/gTvs0fY5s2kJG3gO/arcgis/rest/services/SPE_Article_4_Directions/FeatureServer/1/query?outFields=*&where=1%3D1&f=geojson,,,2023-10-11T12:12:05:56Z,,2024-06-20
-4ae452cf1edffc027a1b929ce7c6e0692a3bd92066bab4853bafa9ecd8907499,https://services-eu1.arcgis.com/xk4RA36G57mVH7Aw/ArcGIS/rest/services/Article_4_National_Planning_Map_view/FeatureServer/0/query?where=1%3D1&f=geojson&outFields=*,,,2023-10-11T15:15:56:44Z,,2023-12-21
-99dd0c0c5c8d8b3d4250ac27ed6e5c04c7a0c661f7f5e698d84c140ac240d1bc,https://maps.epsom-ewell.gov.uk/getOWS.ashx?MapSource=EEBC/planx&service=WFS&version=1.1.0&request=GetFeature&Typename=article4areas,,wfs,2023-10-23T11:11:54:20Z,,
+16857564d1658edeaa7703207b91525d005e7696e2e9392fbb6fa4231fa550ef,https://datamillnorth.org/download/newcastle-article-4-directions/4c650010-8d88-4bba-9f4a-85587742a64d/article4directionareas.csv,,,2023-02-23T22:22:12.390Z,,2024-06-19
+d788f7a3f32b918aca77f75952243261fae0f6eddc6d32636a8955198d0293fd,https://datamillnorth.org/download/newcastle-article-4-directions/5e84e19f-028c-4902-8a23-bd0dd3f388eb/article4directions.csv,,,2023-02-28T11:11:10.230Z,,2024-06-19
+3bbf7138bc9dacbf42700c5d114ba0c1856ae4cdc7c92360e95103ae59d38c0a,https://opendata.camden.gov.uk/api/views/45mg-zjup/rows.csv,,,2023-08-11T16:16:49.180Z,,
+fc6886730a30350775d915a11a8bf5644e42c32395315abd525665a02246751a,https://maps.doncaster.gov.uk/server/rest/services/Planning/Article_4_DSA/MapServer/0,,arcgis,2023-08-30T10:10:04.070Z,,2024-02-05
+22b68af5f133601d5bafc187546b06f0cfffb95a1a6166449cec471ac7ed6706,https://opendata.camden.gov.uk/api/views/ypia-t7j2/rows.csv,,,2023-09-05T10:10:05.410Z,,
+24bc10a266141363496a758b610670d8fbc48495d8cd2e84a362485f0b93e861,https://services7.arcgis.com/NXULfNGIlsKmpCJD/arcgis/rest/services/Article_4_Direction_Area/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson,,,2023-10-09T12:12:45.230Z,,
+d7e334900b5355ff4b43323d146d9d72cd3647bb4b31025aa7bd05dcb69596de,https://services7.arcgis.com/gTvs0fY5s2kJG3gO/arcgis/rest/services/SPE_Article_4_Directions/FeatureServer/1/query?outFields=*&where=1%3D1&f=geojson,,,2023-10-11T12:12:05.560Z,,2024-06-20
+4ae452cf1edffc027a1b929ce7c6e0692a3bd92066bab4853bafa9ecd8907499,https://services-eu1.arcgis.com/xk4RA36G57mVH7Aw/ArcGIS/rest/services/Article_4_National_Planning_Map_view/FeatureServer/0/query?where=1%3D1&f=geojson&outFields=*,,,2023-10-11T15:15:56.440Z,,2023-12-21
+99dd0c0c5c8d8b3d4250ac27ed6e5c04c7a0c661f7f5e698d84c140ac240d1bc,https://maps.epsom-ewell.gov.uk/getOWS.ashx?MapSource=EEBC/planx&service=WFS&version=1.1.0&request=GetFeature&Typename=article4areas,,wfs,2023-10-23T11:11:54.200Z,,
 65672f37a643acac1a9d5599a0631db5d9bad9b2cf8874cc7214f06a712664b2,https://maps.buckscc.gov.uk/arcgis/rest/services/PLANNING/A4D_CSB/MapServer/0/query?outFields=*&where=1%3D1&f=geojson,,wfs,2023-11-07,2023-11-02,
 2d9575d771afff89f6d731be59a1ff8cedfd99efcd8bb2f0a746a2d18c9bd1ca,https://maps.birmingham.gov.uk/server/rest/services/planx/PlanX/FeatureServer/0/query?where=1=1&outfields=*&f=geojson,,wfs,2023-11-14,2023-11-14,2024-09-24
 3139776593c26742a0c9a92d3078957b90799c16cca269dae7229c6152f61e94,https://services3.arcgis.com/lCzPKKaGs7lhrnrV/ArcGIS/rest/services/Planning_Portal_Article_4_Direction_081223/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson,,wfs,2023-12-14,,2025-02-17
 4d69e04b32ecfa83f9c17f1fed6f13a94dc8c839607dd89c6c2fc8bfc2ab5e12,https://open.barnet.gov.uk/download/2ylny/z7y/article4-direction.csv,,,2023-12-18,2023-12-18,
 d4c09389082ca55e33cf532eb045ea7eb9dc447a24547fca734506f7e5f05413,https://open.barnet.gov.uk/download/e5l77/dhv/article4-direction-areas.csv,,,2023-12-18,2023-12-18,
-701a969f68db2fb8bae189534a1f32bb1cc5e357a3f67aa2a7deaae600fd4537,https://services-eu1.arcgis.com/xk4RA36G57mVH7Aw/ArcGIS/rest/services/Article_4_Output_view/FeatureServer/0//query?where=1%3D1&f=geojson&outfields=*,,,2023-12-21T11:11:26:31Z,,
-03b17776d0e7707bdb98768bdcac82cf7d01595a3536032c1c91abf96f2db9f0,https://services-eu1.arcgis.com/xk4RA36G57mVH7Aw/ArcGIS/rest/services/Article_4_Output_view/FeatureServer/1//query?where=1%3D1&f=json&outfields=*,,,2023-12-21T11:11:47:21Z,,
+701a969f68db2fb8bae189534a1f32bb1cc5e357a3f67aa2a7deaae600fd4537,https://services-eu1.arcgis.com/xk4RA36G57mVH7Aw/ArcGIS/rest/services/Article_4_Output_view/FeatureServer/0//query?where=1%3D1&f=geojson&outfields=*,,,2023-12-21T11:11:26.310Z,,
+03b17776d0e7707bdb98768bdcac82cf7d01595a3536032c1c91abf96f2db9f0,https://services-eu1.arcgis.com/xk4RA36G57mVH7Aw/ArcGIS/rest/services/Article_4_Output_view/FeatureServer/1//query?where=1%3D1&f=json&outfields=*,,,2023-12-21T11:11:47.210Z,,
 d1fd06f3e16d52290d3c1f6e6076a4a9129db3eda91bdad28b5f4655e1a80b19,https://tewkesburybc.s3.eu-west-2.amazonaws.com/geojson/article-4-direction-area.geojson,,,2024-01-15,2023-11-03,
 83984124f1346554c4f805f337c1bdee789c375651058eac7ba0a33f83b35167,https://gis.westberks.gov.uk/server/rest/services/Layers/LPA_ARTICLE4_DIRECTIONS/FeatureServer/0/query?where=1%3D1&f=geojson&outFields=*,,,2024-01-22,2023-12-18,
 23a770eea13b2d4345bcaf814aaf37b3ecb386903d0ae1e940b79d5724b31b59,https://maps.doncaster.gov.uk/server/rest/services/Planning/Article_4/MapServer/4/,,arcgis,2024-02-05,2024-02-01,