diff --git a/BinDays.Api.Collectors/Collectors/Councils/TorbayCouncil.cs b/BinDays.Api.Collectors/Collectors/Councils/TorbayCouncil.cs new file mode 100644 index 00000000..b470738d --- /dev/null +++ b/BinDays.Api.Collectors/Collectors/Councils/TorbayCouncil.cs @@ -0,0 +1,305 @@ +namespace BinDays.Api.Collectors.Collectors.Councils; + +using BinDays.Api.Collectors.Collectors.Vendors; +using BinDays.Api.Collectors.Models; +using BinDays.Api.Collectors.Utilities; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Text.RegularExpressions; + +/// +/// Collector implementation for Torbay Council. +/// +internal sealed partial class TorbayCouncil : GovUkCollectorBase, ICollector +{ + /// + public string Name => "Torbay Council"; + + /// + public Uri WebsiteUrl => new("https://www.torbay.gov.uk/recycling/bin-collections/"); + + /// + public override string GovUkId => "torbay"; + + /// + /// The list of bin types for this collector. + /// + private readonly IReadOnlyCollection _binTypes = + [ + new() + { + Name = "General Waste", + Colour = BinColour.Grey, + Type = BinType.Bin, + Keys = [ "Domestic" ], + }, + new() + { + Name = "Plastics & Tins Recycling", + Colour = BinColour.Green, + Type = BinType.Box, + Keys = [ "Recycling" ], + }, + new() + { + Name = "Glass & Cardboard Recycling", + Colour = BinColour.Orange, + Type = BinType.Box, + Keys = [ "Recycling" ], + }, + new() + { + Name = "Paper Recycling", + Colour = BinColour.Blue, + Type = BinType.Bag, + Keys = [ "Recycling" ], + }, + new() + { + Name = "Food Waste", + Colour = BinColour.Brown, + Type = BinType.Bin, + Keys = [ "Recycling" ], + }, + new() + { + Name = "Garden Waste", + Colour = BinColour.Brown, + Type = BinType.Bin, + Keys = [ "Garden" ], + }, + ]; + + /// + /// Regex for the __RequestVerificationToken. + /// + [GeneratedRegex(@"]*name=""__RequestVerificationToken""[^>]*value=""(?[^""]+)""", RegexOptions.IgnoreCase)] + private static partial Regex RequestVerificationTokenRegex(); + + /// + /// Regex for the FormGuid value. + /// + [GeneratedRegex(@"]*name=""FormGuid""[^>]*value=""(?[^""]+)""", RegexOptions.IgnoreCase)] + private static partial Regex FormGuidRegex(); + + /// + /// Regex for the ObjectTemplateID value. + /// + [GeneratedRegex(@"]*name=""ObjectTemplateID""[^>]*value=""(?[^""]+)""", RegexOptions.IgnoreCase)] + private static partial Regex ObjectTemplateIdRegex(); + + /// + /// Regex for bin day rows. + /// + [GeneratedRegex(@"resirow[^>]*>\s*]*class=""col[^""]*""[^>]*>.*?]*class=""col""[^>]*>(?[^<]+)\s*]*class=""col""[^>]*>(?[^<]+) + public GetAddressesResponse GetAddresses(string postcode, ClientSideResponse? clientSideResponse) + { + // Prepare client-side request for getting form tokens and cookies + if (clientSideResponse == null) + { + var clientSideRequest = new ClientSideRequest + { + RequestId = 1, + Url = "https://selfservice-torbay.servicebuilder.co.uk/renderform?t=62&k=09B72FF904A21A4B01A72AB6CCF28DC95105031C", + Method = "GET", + Headers = new() + { + { "User-Agent", Constants.UserAgent }, + }, + }; + + var getAddressesResponse = new GetAddressesResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getAddressesResponse; + } + // Prepare client-side request for getting addresses + else if (clientSideResponse.RequestId == 1) + { + var requestCookies = ProcessingUtilities.ParseSetCookieHeaderForRequestCookie(clientSideResponse.Headers["set-cookie"]); + + var requestBody = ProcessingUtilities.ConvertDictionaryToFormData(new() + { + { "query", postcode }, + { "searchNlpg", "False" }, + { "classification", string.Empty }, + }); + + var clientSideRequest = new ClientSideRequest + { + RequestId = 2, + Url = "https://selfservice-torbay.servicebuilder.co.uk/core/addresslookup", + Method = "POST", + Headers = new() + { + { "User-Agent", Constants.UserAgent }, + { "Content-Type", "application/x-www-form-urlencoded" }, + { "Cookie", requestCookies }, + { "x-requested-with", "XMLHttpRequest" }, + }, + Body = requestBody, + }; + + var getAddressesResponse = new GetAddressesResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getAddressesResponse; + } + // Process addresses from response + else if (clientSideResponse.RequestId == 2) + { + using var document = JsonDocument.Parse(clientSideResponse.Content); + + // Iterate through each address, and create a new address object + var addresses = new List
(); + foreach (var element in document.RootElement.EnumerateArray()) + { + var uid = element.GetProperty("Key").GetString(); + var addressText = element.GetProperty("Value").GetString(); + + var address = new Address + { + Property = addressText?.Trim(), + Postcode = postcode, + Uid = uid, + }; + + addresses.Add(address); + } + + var getAddressesResponse = new GetAddressesResponse + { + Addresses = [.. addresses], + }; + + return getAddressesResponse; + } + + // Throw exception for invalid request + throw new InvalidOperationException("Invalid client-side request."); + } + + /// + public GetBinDaysResponse GetBinDays(Address address, ClientSideResponse? clientSideResponse) + { + // Prepare client-side request for getting form tokens and cookies + if (clientSideResponse == null) + { + var clientSideRequest = new ClientSideRequest + { + RequestId = 1, + Url = "https://selfservice-torbay.servicebuilder.co.uk/renderform?t=62&k=09B72FF904A21A4B01A72AB6CCF28DC95105031C", + Method = "GET", + Headers = new() + { + { "User-Agent", Constants.UserAgent }, + }, + }; + + var getBinDaysResponse = new GetBinDaysResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getBinDaysResponse; + } + // Prepare client-side request for getting bin days + else if (clientSideResponse.RequestId == 1) + { + var requestCookies = ProcessingUtilities.ParseSetCookieHeaderForRequestCookie(clientSideResponse.Headers["set-cookie"]); + + var token = RequestVerificationTokenRegex().Match(clientSideResponse.Content).Groups["token"].Value; + var formGuid = FormGuidRegex().Match(clientSideResponse.Content).Groups["formGuid"].Value; + var objectTemplateId = ObjectTemplateIdRegex().Match(clientSideResponse.Content).Groups["objectTemplateId"].Value; + + if (string.IsNullOrWhiteSpace(token) || string.IsNullOrWhiteSpace(formGuid) || string.IsNullOrWhiteSpace(objectTemplateId)) + { + throw new InvalidOperationException("Failed to extract one or more form tokens from the page. The council website may have changed."); + } + + var requestBody = ProcessingUtilities.ConvertDictionaryToFormData(new() + { + { "__RequestVerificationToken", token }, + { "FormGuid", formGuid }, + { "ObjectTemplateID", objectTemplateId }, + { "Trigger", "submit" }, + { "CurrentSectionID", "0" }, + { "TriggerCtl", string.Empty }, + { "FF1168", address.Uid! }, + { "FF1168lbltxt", "Please select your address" }, + { "FF1168-text", address.Postcode! }, + }); + + var clientSideRequest = new ClientSideRequest + { + RequestId = 2, + Url = "https://selfservice-torbay.servicebuilder.co.uk/renderform/Form", + Method = "POST", + Headers = new() + { + { "User-Agent", Constants.UserAgent }, + { "Content-Type", "application/x-www-form-urlencoded" }, + { "Cookie", requestCookies }, + { "x-requested-with", "XMLHttpRequest" }, + }, + Body = requestBody, + }; + + var getBinDaysResponse = new GetBinDaysResponse + { + NextClientSideRequest = clientSideRequest, + }; + + return getBinDaysResponse; + } + // Process bin days from response + else if (clientSideResponse.RequestId == 2) + { + // Iterate through each bin day row, and create a new bin day object + var binDays = new List(); + foreach (Match match in BinDayRegex().Matches(clientSideResponse.Content)!) + { + var dateString = match.Groups["date"].Value.Trim(); + var service = match.Groups["service"].Value.Trim(); + + var date = DateOnly.ParseExact( + dateString, + "dddd dd MMMM yyyy", + CultureInfo.InvariantCulture, + DateTimeStyles.None + ); + + var bins = ProcessingUtilities.GetMatchingBins(_binTypes, service); + + var binDay = new BinDay + { + Date = date, + Address = address, + Bins = bins, + }; + + binDays.Add(binDay); + } + + var getBinDaysResponse = new GetBinDaysResponse + { + BinDays = ProcessingUtilities.ProcessBinDays(binDays), + }; + + return getBinDaysResponse; + } + + // Throw exception for invalid request + throw new InvalidOperationException("Invalid client-side request."); + } +} diff --git a/BinDays.Api.IntegrationTests/Collectors/Councils/TorbayCouncilTests.cs b/BinDays.Api.IntegrationTests/Collectors/Councils/TorbayCouncilTests.cs new file mode 100644 index 00000000..85f2b392 --- /dev/null +++ b/BinDays.Api.IntegrationTests/Collectors/Councils/TorbayCouncilTests.cs @@ -0,0 +1,36 @@ +namespace BinDays.Api.IntegrationTests.Collectors.Councils; + +using BinDays.Api.Collectors.Collectors; +using BinDays.Api.Collectors.Collectors.Councils; +using BinDays.Api.Collectors.Services; +using BinDays.Api.IntegrationTests.Helpers; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +public class TorbayCouncilTests +{ + private readonly IntegrationTestClient _client; + private static readonly ICollector _collector = new TorbayCouncil(); + private readonly CollectorService _collectorService = new([_collector]); + private readonly ITestOutputHelper _outputHelper; + + public TorbayCouncilTests(ITestOutputHelper outputHelper) + { + _outputHelper = outputHelper; + _client = new IntegrationTestClient(outputHelper); + } + + [Theory] + [InlineData("TQ1 1NX")] + public async Task GetBinDaysTest(string postcode) + { + await TestSteps.EndToEnd( + _client, + _collectorService, + _collector, + postcode, + _outputHelper + ); + } +}