From fe67a0a8a827cabf641976e454567964e09f08ed Mon Sep 17 00:00:00 2001 From: BadgerHobbs Date: Sat, 17 Jan 2026 00:17:26 +0000 Subject: [PATCH 1/2] Add collector for CheshireEastCouncil Closes #83 Generated with Codex CLI --- .../Councils/CheshireEastCouncil.cs | 257 ++++++++++++++++++ .../Councils/CheshireEastCouncilTests.cs | 36 +++ 2 files changed, 293 insertions(+) create mode 100644 BinDays.Api.Collectors/Collectors/Councils/CheshireEastCouncil.cs create mode 100644 BinDays.Api.IntegrationTests/Collectors/Councils/CheshireEastCouncilTests.cs diff --git a/BinDays.Api.Collectors/Collectors/Councils/CheshireEastCouncil.cs b/BinDays.Api.Collectors/Collectors/Councils/CheshireEastCouncil.cs new file mode 100644 index 00000000..6dfe613e --- /dev/null +++ b/BinDays.Api.Collectors/Collectors/Councils/CheshireEastCouncil.cs @@ -0,0 +1,257 @@ +namespace BinDays.Api.Collectors.Collectors.Councils; + +using BinDays.Api.Collectors.Collectors.Vendors; +using BinDays.Api.Collectors.Models; +using BinDays.Api.Collectors.Utilities; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.RegularExpressions; + +/// +/// Collector implementation for Cheshire East Council. +/// +internal sealed partial class CheshireEastCouncil : GovUkCollectorBase, ICollector +{ + /// + public string Name => "Cheshire East Council"; + + /// + public Uri WebsiteUrl => new("https://online.cheshireeast.gov.uk/"); + + /// + public override string GovUkId => "cheshire-east"; + + /// + /// The list of bin types for this collector. + /// + private readonly IReadOnlyCollection _binTypes = + [ + new() + { + Name = "General waste", + Colour = BinColour.Black, + Keys = [ "General Waste" ], + Type = BinType.Bin, + }, + new() + { + Name = "Recycling", + Colour = BinColour.Grey, + Keys = [ "Mixed Recycling" ], + Type = BinType.Bin, + }, + new() + { + Name = "Garden waste", + Colour = BinColour.Green, + Keys = [ "Garden Waste" ], + Type = BinType.Bin, + }, + ]; + + /// + /// Regex for parsing addresses from the search response. + /// + [GeneratedRegex(@"]*data-uprn=""(?[^""]+)""[^>]*>(?
[^<]+)", RegexOptions.IgnoreCase)] + private static partial Regex AddressRegex(); + + /// + /// Regex for parsing bin day rows from the job list. + /// + [GeneratedRegex(@"BartecSimplifiedJobList_(?\d+)__Name""[^>]*value=""(?[^""]+)"" .*?BartecSimplifiedJobList_\k__ScheduledStart""[^>]*value=""(?[^""]+)""", RegexOptions.Singleline)] + private static partial Regex BinDayRegex(); + + /// + public GetAddressesResponse GetAddresses(string postcode, ClientSideResponse? clientSideResponse) + { + // Prepare client-side request for getting session cookies + if (clientSideResponse == null) + { + var clientSideRequest = new ClientSideRequest + { + RequestId = 1, + Url = "https://online.cheshireeast.gov.uk/MyCollectionDay/", + Method = "GET", + Headers = new() + { + { "User-Agent", Constants.UserAgent }, + }, + }; + + var getAddressesResponse = new GetAddressesResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getAddressesResponse; + } + // Prepare client-side request for getting addresses + else if (clientSideResponse.RequestId == 1) + { + var formattedPostcode = ProcessingUtilities.FormatPostcode(postcode); + var timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var requestCookies = ProcessingUtilities.ParseSetCookieHeaderForRequestCookie( + clientSideResponse.Headers.GetValueOrDefault("set-cookie") ?? string.Empty); + var requestUrl = $"https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/Search?postcode={Uri.EscapeDataString(formattedPostcode)}&propertyname=&_={timestamp}"; + + var clientSideRequest = new ClientSideRequest + { + RequestId = 2, + Url = requestUrl, + Method = "GET", + Headers = new() + { + { "User-Agent", Constants.UserAgent }, + { "x-requested-with", "XMLHttpRequest" }, + { "cookie", requestCookies }, + }, + }; + + var getAddressesResponse = new GetAddressesResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getAddressesResponse; + } + // Process addresses from response + else if (clientSideResponse.RequestId == 2) + { + var rawAddresses = AddressRegex().Matches(clientSideResponse.Content); + var formattedPostcode = ProcessingUtilities.FormatPostcode(postcode); + + // Iterate through each address, and create a new address object + var addresses = new List
(); + foreach (Match rawAddress in rawAddresses) + { + var addressText = rawAddress.Groups["address"].Value.Trim(); + var addressParts = addressText.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + var address = new Address + { + Property = addressParts.Length > 0 ? addressParts[0] : null, + Street = addressParts.Length > 3 ? addressParts[1] : null, + Town = addressParts.Length > 3 ? addressParts[2] : addressParts.Length > 2 ? addressParts[1] : null, + Postcode = addressParts.Length > 0 ? addressParts[^1] : formattedPostcode, + Uid = rawAddress.Groups["uprn"].Value, + }; + + addresses.Add(address); + } + + var getAddressesResponse = new GetAddressesResponse + { + Addresses = [.. addresses], + }; + + return getAddressesResponse; + } + + // Throw exception for invalid request + throw new InvalidOperationException("Invalid client-side request."); + } + + /// + public GetBinDaysResponse GetBinDays(Address address, ClientSideResponse? clientSideResponse) + { + // Prepare client-side request for getting session cookies + if (clientSideResponse == null) + { + var clientSideRequest = new ClientSideRequest + { + RequestId = 1, + Url = "https://online.cheshireeast.gov.uk/MyCollectionDay/", + Method = "GET", + Headers = new() + { + { "User-Agent", Constants.UserAgent }, + }, + }; + + var getBinDaysResponse = new GetBinDaysResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getBinDaysResponse; + } + // Prepare client-side request for getting bin days + else if (clientSideResponse.RequestId == 1) + { + var formattedPostcode = ProcessingUtilities.FormatPostcode(address.Postcode ?? string.Empty); + var onelineAddress = address.Property ?? address.Street ?? address.Town ?? string.Empty; + if (!string.IsNullOrWhiteSpace(formattedPostcode)) + { + onelineAddress = string.IsNullOrWhiteSpace(onelineAddress) + ? formattedPostcode + : $"{onelineAddress}, {formattedPostcode}"; + } + + var timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var requestCookies = ProcessingUtilities.ParseSetCookieHeaderForRequestCookie( + clientSideResponse.Headers.GetValueOrDefault("set-cookie") ?? string.Empty); + var requestUrl = $"https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/GetBartecJobList?uprn={address.Uid}&onelineaddress={Uri.EscapeDataString(onelineAddress)}&_={timestamp}"; + + var clientSideRequest = new ClientSideRequest + { + RequestId = 2, + Url = requestUrl, + Method = "GET", + Headers = new() + { + { "User-Agent", Constants.UserAgent }, + { "x-requested-with", "XMLHttpRequest" }, + { "cookie", requestCookies }, + }, + }; + + var getBinDaysResponse = new GetBinDaysResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getBinDaysResponse; + } + // Process bin days from response + else if (clientSideResponse.RequestId == 2) + { + var rawBinDays = BinDayRegex().Matches(clientSideResponse.Content); + + // Iterate through each bin day, and create a new bin day object + var binDays = new List(); + foreach (Match rawBinDay in rawBinDays) + { + var binTypeStr = rawBinDay.Groups["name"].Value.Trim(); + var dateStr = rawBinDay.Groups["date"].Value.Trim(); + + // Parse date string (e.g. "13/01/2026 07:00:00") + var dateTime = DateTime.ParseExact( + dateStr, + "dd/MM/yyyy HH:mm:ss", + CultureInfo.InvariantCulture, + DateTimeStyles.None + ); + + var binDay = new BinDay + { + Date = DateOnly.FromDateTime(dateTime), + Address = address, + Bins = ProcessingUtilities.GetMatchingBins(_binTypes, binTypeStr), + }; + + binDays.Add(binDay); + } + + var getBinDaysResponse = new GetBinDaysResponse + { + BinDays = ProcessingUtilities.ProcessBinDays(binDays), + }; + + return getBinDaysResponse; + } + + // Throw exception for invalid request + throw new InvalidOperationException("Invalid client-side request."); + } +} diff --git a/BinDays.Api.IntegrationTests/Collectors/Councils/CheshireEastCouncilTests.cs b/BinDays.Api.IntegrationTests/Collectors/Councils/CheshireEastCouncilTests.cs new file mode 100644 index 00000000..c1f54b76 --- /dev/null +++ b/BinDays.Api.IntegrationTests/Collectors/Councils/CheshireEastCouncilTests.cs @@ -0,0 +1,36 @@ +namespace BinDays.Api.IntegrationTests.Collectors.Councils; + +using BinDays.Api.Collectors.Collectors; +using BinDays.Api.Collectors.Collectors.Councils; +using BinDays.Api.Collectors.Services; +using BinDays.Api.IntegrationTests.Helpers; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +public class CheshireEastCouncilTests +{ + private readonly IntegrationTestClient _client; + private static readonly ICollector _collector = new CheshireEastCouncil(); + private readonly CollectorService _collectorService = new([_collector]); + private readonly ITestOutputHelper _outputHelper; + + public CheshireEastCouncilTests(ITestOutputHelper outputHelper) + { + _outputHelper = outputHelper; + _client = new IntegrationTestClient(outputHelper); + } + + [Theory] + [InlineData("CW1 2AY")] + public async Task GetBinDaysTest(string postcode) + { + await TestSteps.EndToEnd( + _client, + _collectorService, + _collector, + postcode, + _outputHelper + ); + } +} From 37d47828b5593fa001b5110cca5b8f6847f93ec2 Mon Sep 17 00:00:00 2001 From: BadgerHobbs Date: Mon, 19 Jan 2026 22:23:07 +0000 Subject: [PATCH 2/2] Address PR #95 review comments - Remove redundant postcode formatting (already formatted) - Remove timestamp cache-busting parameters (not required) - Remove URI escaping (not needed) - Simplify Address object to use Property and Postcode only - Expect postcode and cookies to be provided (remove conditionals) - Use postcode from request parameter, not parsed address parts - Fix null-forgiving operators per style guide (on Matches, not Groups) - Add comment for onelineAddress building logic Co-Authored-By: Claude Sonnet 4.5 --- .../Councils/CheshireEastCouncil.cs | 37 ++++++------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/BinDays.Api.Collectors/Collectors/Councils/CheshireEastCouncil.cs b/BinDays.Api.Collectors/Collectors/Councils/CheshireEastCouncil.cs index 6dfe613e..a4092579 100644 --- a/BinDays.Api.Collectors/Collectors/Councils/CheshireEastCouncil.cs +++ b/BinDays.Api.Collectors/Collectors/Councils/CheshireEastCouncil.cs @@ -89,11 +89,9 @@ public GetAddressesResponse GetAddresses(string postcode, ClientSideResponse? cl // Prepare client-side request for getting addresses else if (clientSideResponse.RequestId == 1) { - var formattedPostcode = ProcessingUtilities.FormatPostcode(postcode); - var timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - var requestCookies = ProcessingUtilities.ParseSetCookieHeaderForRequestCookie( - clientSideResponse.Headers.GetValueOrDefault("set-cookie") ?? string.Empty); - var requestUrl = $"https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/Search?postcode={Uri.EscapeDataString(formattedPostcode)}&propertyname=&_={timestamp}"; + var setCookie = clientSideResponse.Headers["set-cookie"]; + var requestCookies = ProcessingUtilities.ParseSetCookieHeaderForRequestCookie(setCookie); + var requestUrl = $"https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/Search?postcode={postcode}&propertyname="; var clientSideRequest = new ClientSideRequest { @@ -118,22 +116,18 @@ public GetAddressesResponse GetAddresses(string postcode, ClientSideResponse? cl // Process addresses from response else if (clientSideResponse.RequestId == 2) { - var rawAddresses = AddressRegex().Matches(clientSideResponse.Content); - var formattedPostcode = ProcessingUtilities.FormatPostcode(postcode); + var rawAddresses = AddressRegex().Matches(clientSideResponse.Content)!; // Iterate through each address, and create a new address object var addresses = new List
(); foreach (Match rawAddress in rawAddresses) { var addressText = rawAddress.Groups["address"].Value.Trim(); - var addressParts = addressText.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); var address = new Address { - Property = addressParts.Length > 0 ? addressParts[0] : null, - Street = addressParts.Length > 3 ? addressParts[1] : null, - Town = addressParts.Length > 3 ? addressParts[2] : addressParts.Length > 2 ? addressParts[1] : null, - Postcode = addressParts.Length > 0 ? addressParts[^1] : formattedPostcode, + Property = addressText, + Postcode = postcode, Uid = rawAddress.Groups["uprn"].Value, }; @@ -179,19 +173,12 @@ public GetBinDaysResponse GetBinDays(Address address, ClientSideResponse? client // Prepare client-side request for getting bin days else if (clientSideResponse.RequestId == 1) { - var formattedPostcode = ProcessingUtilities.FormatPostcode(address.Postcode ?? string.Empty); - var onelineAddress = address.Property ?? address.Street ?? address.Town ?? string.Empty; - if (!string.IsNullOrWhiteSpace(formattedPostcode)) - { - onelineAddress = string.IsNullOrWhiteSpace(onelineAddress) - ? formattedPostcode - : $"{onelineAddress}, {formattedPostcode}"; - } + // Build the full address string for the API request + var onelineAddress = $"{address.Property}, {address.Postcode}"; - var timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - var requestCookies = ProcessingUtilities.ParseSetCookieHeaderForRequestCookie( - clientSideResponse.Headers.GetValueOrDefault("set-cookie") ?? string.Empty); - var requestUrl = $"https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/GetBartecJobList?uprn={address.Uid}&onelineaddress={Uri.EscapeDataString(onelineAddress)}&_={timestamp}"; + var setCookie = clientSideResponse.Headers["set-cookie"]; + var requestCookies = ProcessingUtilities.ParseSetCookieHeaderForRequestCookie(setCookie); + var requestUrl = $"https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/GetBartecJobList?uprn={address.Uid}&onelineaddress={onelineAddress}"; var clientSideRequest = new ClientSideRequest { @@ -216,7 +203,7 @@ public GetBinDaysResponse GetBinDays(Address address, ClientSideResponse? client // Process bin days from response else if (clientSideResponse.RequestId == 2) { - var rawBinDays = BinDayRegex().Matches(clientSideResponse.Content); + var rawBinDays = BinDayRegex().Matches(clientSideResponse.Content)!; // Iterate through each bin day, and create a new bin day object var binDays = new List();