diff --git a/BinDays.Api.Collectors/Collectors/Councils/LondonBoroughOfWalthamForest.cs b/BinDays.Api.Collectors/Collectors/Councils/LondonBoroughOfWalthamForest.cs new file mode 100644 index 0000000..bdb52ae --- /dev/null +++ b/BinDays.Api.Collectors/Collectors/Councils/LondonBoroughOfWalthamForest.cs @@ -0,0 +1,469 @@ +namespace BinDays.Api.Collectors.Collectors.Councils; + +using BinDays.Api.Collectors.Collectors.Vendors; +using BinDays.Api.Collectors.Models; +using BinDays.Api.Collectors.Utilities; +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.RegularExpressions; + +/// +/// Collector implementation for London Borough of Waltham Forest. +/// +internal sealed partial class LondonBoroughOfWalthamForest : GovUkCollectorBase, ICollector +{ + /// + public string Name => "London Borough of Waltham Forest"; + + /// + public Uri WebsiteUrl => new("https://www.walthamforest.gov.uk/rubbish-and-recycling/household-bin-collections/check-your-collection-days"); + + /// + public override string GovUkId => "waltham-forest"; + + /// + /// The list of bin types for this collector. + /// + private readonly IReadOnlyCollection _binTypes = + [ + new() + { + Name = "General Waste", + Colour = BinColour.Black, + Keys = [ "Domestic Waste Collection Service" ], + }, + new() + { + Name = "Recycling", + Colour = BinColour.Green, + Keys = [ "Recycling Collection Service" ], + }, + new() + { + Name = "Garden Waste", + Colour = BinColour.Brown, + Keys = [ "Garden Waste Collection Service" ], + }, + new() + { + Name = "Food Waste", + Colour = BinColour.Brown, + Keys = [ "Food Waste Collection Service" ], + Type = BinType.Caddy, + }, + ]; + + /// + /// The AchieveForms URL for the bin collection lookup form. + /// + private const string _achieveFormsUrl = "https://portal.walthamforest.gov.uk/AchieveForms/?mode=fill&consentMessage=yes&form_uri=sandbox-publish://AF-Process-d62ccdd2-3de9-48eb-a229-8e20cbdd6393/AF-Stage-8bf39bf9-5391-4c24-857f-0dc2025c67f4/definition.json&process=1&process_uri=sandbox-processes://AF-Process-d62ccdd2-3de9-48eb-a229-8e20cbdd6393&process_id=AF-Process-d62ccdd2-3de9-48eb-a229-8e20cbdd6393"; + + /// + /// The base URL for API broker requests. + /// + private const string _apibrokerBaseUrl = "https://portal.walthamforest.gov.uk/apibroker/"; + + /// + /// The stage ID for the form workflow. + /// + private const string _stageId = "AF-Stage-8bf39bf9-5391-4c24-857f-0dc2025c67f4"; + + /// + /// The form ID for address lookup. + /// + private const string _addressFormId = "AF-Form-08647570-43e4-4e68-9d6a-65d914e27ef7"; + + /// + /// The form ID for the main bin collection form. + /// + private const string _mainFormId = "AF-Form-07a98da7-bc6b-4df6-aa46-33a06312acce"; + + /// + /// The lookup ID for address search. + /// + private const string _addressLookupId = "5694fd42a5541"; + + /// + /// Regex to extract the session ID (sid) from HTML content. + /// + [GeneratedRegex(@"sid=([a-f0-9]+)")] + private static partial Regex SessionIdRegex(); + + /// + public GetAddressesResponse GetAddresses(string postcode, ClientSideResponse? clientSideResponse) + { + // Prepare client-side request for getting addresses + if (clientSideResponse == null) + { + _ = ProcessingUtilities.FormatPostcode(postcode); + + var clientSideRequest = new ClientSideRequest + { + RequestId = 1, + Url = _achieveFormsUrl, + Method = "GET", + Headers = new() + { + { "user-agent", Constants.UserAgent }, + }, + }; + + var getAddressesResponse = new GetAddressesResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getAddressesResponse; + } + // Prepare client-side request for address lookup + else if (clientSideResponse.RequestId == 1) + { + var sid = SessionIdRegex().Match(clientSideResponse.Content).Groups[1].Value; + var requestCookies = ProcessingUtilities.ParseSetCookieHeaderForRequestCookie( + clientSideResponse.Headers["set-cookie"] + ); + var formattedPostcode = ProcessingUtilities.FormatPostcode(postcode); + + var requestBody = BuildAddressLookupPayload(formattedPostcode); + + var clientSideRequest = new ClientSideRequest + { + RequestId = 2, + Url = $"{_apibrokerBaseUrl}?api=RunLookup&id={_addressLookupId}&repeat_against=&noRetry=false&getOnlyTokens=undefined&log_id=&app_name=AF-Renderer::Self&sid={sid}", + Method = "POST", + Headers = new() + { + { "content-type", "application/json" }, + { "cookie", requestCookies }, + { "x-requested-with", "XMLHttpRequest" }, + }, + Body = requestBody, + }; + + var getAddressesResponse = new GetAddressesResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getAddressesResponse; + } + // Process addresses from response + else if (clientSideResponse.RequestId == 2) + { + using var jsonDoc = JsonDocument.Parse(clientSideResponse.Content); + var rowsData = jsonDoc.RootElement + .GetProperty("integration") + .GetProperty("transformed") + .GetProperty("rows_data"); + + // Iterate through each address, and create a new address object + var addresses = new List
(); + foreach (var property in rowsData.EnumerateObject()) + { + var addressData = property.Value; + var display = addressData.GetProperty("display").GetString()!; + var uprn = addressData.GetProperty("overview_uprn").GetString()!; + var addressPostcode = addressData.GetProperty("overview_postcode").GetString()!; + + var address = new Address + { + Property = display.Trim(), + Postcode = addressPostcode.Trim(), + Uid = uprn, + }; + + addresses.Add(address); + } + + var getAddressesResponse = new GetAddressesResponse + { + Addresses = [.. addresses], + }; + + return getAddressesResponse; + } + + throw new InvalidOperationException("Invalid client-side request."); + } + + /// + public GetBinDaysResponse GetBinDays(Address address, ClientSideResponse? clientSideResponse) + { + // Prepare client-side request for getting bin days + if (clientSideResponse == null) + { + var clientSideRequest = new ClientSideRequest + { + RequestId = 1, + Url = _achieveFormsUrl, + Method = "GET", + Headers = new() + { + { "user-agent", Constants.UserAgent }, + }, + Options = new ClientSideOptions + { + Metadata = + { + { "uprn", address.Uid! }, + }, + }, + }; + + var getBinDaysResponse = new GetBinDaysResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getBinDaysResponse; + } + // Prepare client-side request for address lookup before bin days + else if (clientSideResponse.RequestId == 1) + { + var sid = SessionIdRegex().Match(clientSideResponse.Content).Groups[1].Value; + var requestCookies = ProcessingUtilities.ParseSetCookieHeaderForRequestCookie( + clientSideResponse.Headers["set-cookie"] + ); + var formattedPostcode = ProcessingUtilities.FormatPostcode(address.Postcode!); + + var requestBody = BuildAddressLookupPayload(formattedPostcode); + + var clientSideRequest = new ClientSideRequest + { + RequestId = 2, + Url = $"{_apibrokerBaseUrl}?api=RunLookup&id={_addressLookupId}&repeat_against=&noRetry=false&getOnlyTokens=undefined&log_id=&app_name=AF-Renderer::Self&sid={sid}", + Method = "POST", + Headers = new() + { + { "content-type", "application/json" }, + { "cookie", requestCookies }, + { "x-requested-with", "XMLHttpRequest" }, + }, + Body = requestBody, + Options = new ClientSideOptions + { + Metadata = + { + { "cookie", requestCookies }, + { "sid", sid }, + { "uprn", address.Uid! }, + }, + }, + }; + + var getBinDaysResponse = new GetBinDaysResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getBinDaysResponse; + } + // Prepare client-side request for bin collections lookup + else if (clientSideResponse.RequestId == 2) + { + using var jsonDoc = JsonDocument.Parse(clientSideResponse.Content); + var rowsData = jsonDoc.RootElement + .GetProperty("integration") + .GetProperty("transformed") + .GetProperty("rows_data"); + + var uprn = clientSideResponse.Options.Metadata["uprn"]; + var matchedAddress = rowsData.EnumerateObject() + .Select(p => p.Value) + .FirstOrDefault(v => v.GetProperty("overview_uprn").GetString()! == uprn); + + var sid = clientSideResponse.Options.Metadata["sid"]; + var requestCookies = clientSideResponse.Options.Metadata["cookie"]; + + var addressDisplay = matchedAddress.GetProperty("display").GetString()!; + var ward = matchedAddress.GetProperty("overview_ward").GetString()!; + var formattedPostcode = matchedAddress.GetProperty("overview_postcode").GetString()!; + + var requestBody = BuildMainFormPayload(formattedPostcode, uprn, addressDisplay, ward); + + var clientSideRequest = new ClientSideRequest + { + RequestId = 3, + Url = $"{_apibrokerBaseUrl}runLookup?id=5e42e28b44d9e&repeat_against=&noRetry=true&getOnlyTokens=undefined&log_id=&app_name=AF-Renderer::Self&sid={sid}", + Method = "POST", + Headers = new() + { + { "content-type", "application/json" }, + { "cookie", requestCookies }, + { "x-requested-with", "XMLHttpRequest" }, + }, + Body = requestBody, + Options = new ClientSideOptions + { + Metadata = + { + { "cookie", requestCookies }, + { "sid", sid }, + { "uprn", uprn }, + { "address", addressDisplay }, + { "ward", ward }, + { "postcode", formattedPostcode }, + }, + }, + }; + + var getBinDaysResponse = new GetBinDaysResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getBinDaysResponse; + } + // Process bin days from response + else if (clientSideResponse.RequestId == 3) + { + var sid = clientSideResponse.Options.Metadata["sid"]; + var requestCookies = clientSideResponse.Options.Metadata["cookie"]; + var uprn = clientSideResponse.Options.Metadata["uprn"]; + var addressDisplay = clientSideResponse.Options.Metadata["address"]; + var ward = clientSideResponse.Options.Metadata["ward"]; + var formattedPostcode = clientSideResponse.Options.Metadata["postcode"]; + + var requestBody = BuildMainFormPayload(formattedPostcode, uprn, addressDisplay, ward); + + var clientSideRequest = new ClientSideRequest + { + RequestId = 4, + Url = $"{_apibrokerBaseUrl}runLookup?id=5e208cda0d0a0&repeat_against=&noRetry=false&getOnlyTokens=undefined&log_id=&app_name=AF-Renderer::Self&sid={sid}", + Method = "POST", + Headers = new() + { + { "content-type", "application/json" }, + { "cookie", requestCookies }, + { "x-requested-with", "XMLHttpRequest" }, + }, + Body = requestBody, + }; + + var getBinDaysResponse = new GetBinDaysResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getBinDaysResponse; + } + // Process bin day results + else if (clientSideResponse.RequestId == 4) + { + using var jsonDoc = JsonDocument.Parse(clientSideResponse.Content); + var rowsData = jsonDoc.RootElement + .GetProperty("integration") + .GetProperty("transformed") + .GetProperty("rows_data"); + + // Iterate through each bin day record, and create a new bin day object + var binDays = new List(); + var binDayEntries = rowsData.ValueKind == JsonValueKind.Object + ? rowsData.EnumerateObject().Select(p => p.Value) + : rowsData.EnumerateArray(); + + foreach (var binData in binDayEntries) + { + var serviceName = binData.GetProperty("ServiceName").GetString()!; + var nextCollectionDate = binData.GetProperty("NextCollectionDate").GetString()!.Trim(); + + if (string.IsNullOrWhiteSpace(nextCollectionDate) || nextCollectionDate.Contains("NaN", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var date = nextCollectionDate.ParseDateInferringYear("dddd d MMMM"); + var matchedBins = ProcessingUtilities.GetMatchingBins(_binTypes, serviceName); + + var binDay = new BinDay + { + Date = date, + Address = address, + Bins = matchedBins, + }; + + binDays.Add(binDay); + } + + var getBinDaysResponse = new GetBinDaysResponse + { + BinDays = ProcessingUtilities.ProcessBinDays(binDays), + }; + + return getBinDaysResponse; + } + + throw new InvalidOperationException("Invalid client-side request."); + } + + /// + /// Builds the address lookup request payload for postcode search. + /// + /// The formatted postcode to search for. + /// JSON payload as a string. + private static string BuildAddressLookupPayload(string postcode) + { + return $$""" + { + "formId": "{{_addressFormId}}", + "formValues": { + "Section 1": { + "postcode_search": { + "value": "{{postcode}}" + } + } + } + } + """; + } + + /// + /// Builds the main form request payload for bin collection lookups. + /// + /// The formatted postcode. + /// The unique property reference number. + /// The full display address. + /// The ward name. + /// JSON payload as a string. + private static string BuildMainFormPayload(string postcode, string uprn, string addressDisplay, string ward) + { + return $$""" + { + "formId": "{{_mainFormId}}", + "formValues": { + "Section 1": { + "calcWardCode": { + "value": "{{ward}}" + }, + "addressLookup": { + "value": { + "Section 1": { + "postcode_search": { + "value": "{{postcode}}" + }, + "postcodeFound": { + "value": "1" + }, + "YourAddress": { + "value_label": ["{{addressDisplay}}"], + "value": "{{uprn}}" + }, + "uprnConfirm": { + "value": "{{uprn}}" + }, + "wardName": { + "value": "{{ward}}" + } + } + } + }, + "inputUPRN": { + "value": "{{uprn}}" + } + } + } + } + """; + } +} diff --git a/BinDays.Api.IntegrationTests/Collectors/Councils/LondonBoroughOfWalthamForestTests.cs b/BinDays.Api.IntegrationTests/Collectors/Councils/LondonBoroughOfWalthamForestTests.cs new file mode 100644 index 0000000..9f01d86 --- /dev/null +++ b/BinDays.Api.IntegrationTests/Collectors/Councils/LondonBoroughOfWalthamForestTests.cs @@ -0,0 +1,30 @@ +namespace BinDays.Api.IntegrationTests.Collectors.Councils; + +using BinDays.Api.IntegrationTests.Helpers; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +public class LondonBoroughOfWalthamForestTests +{ + private readonly IntegrationTestClient _client; + private readonly ITestOutputHelper _outputHelper; + + public LondonBoroughOfWalthamForestTests(ITestOutputHelper outputHelper) + { + _outputHelper = outputHelper; + _client = new IntegrationTestClient(outputHelper); + } + + [Theory] + [InlineData("E17 9HN")] + public async Task GetBinDaysTest(string postcode) + { + await TestSteps.EndToEnd( + _client, + postcode, + "waltham-forest", + _outputHelper + ); + } +}