]*class=""col[^""]*""[^>]*>.*?
]*class=""col""[^>]*>(?[^<]+)
\s*
]*class=""col""[^>]*>(?[^<]+)
+ public GetAddressesResponse GetAddresses(string postcode, ClientSideResponse? clientSideResponse)
+ {
+ // Prepare client-side request for getting form tokens and cookies
+ if (clientSideResponse == null)
+ {
+ var clientSideRequest = new ClientSideRequest
+ {
+ RequestId = 1,
+ Url = "https://selfservice-torbay.servicebuilder.co.uk/renderform?t=62&k=09B72FF904A21A4B01A72AB6CCF28DC95105031C",
+ Method = "GET",
+ Headers = new()
+ {
+ { "User-Agent", Constants.UserAgent },
+ },
+ };
+
+ var getAddressesResponse = new GetAddressesResponse
+ {
+ NextClientSideRequest = clientSideRequest,
+ };
+
+ return getAddressesResponse;
+ }
+ // Prepare client-side request for getting addresses
+ else if (clientSideResponse.RequestId == 1)
+ {
+ var requestCookies = ProcessingUtilities.ParseSetCookieHeaderForRequestCookie(clientSideResponse.Headers["set-cookie"]);
+
+ var requestBody = ProcessingUtilities.ConvertDictionaryToFormData(new()
+ {
+ { "query", postcode },
+ { "searchNlpg", "False" },
+ { "classification", string.Empty },
+ });
+
+ var clientSideRequest = new ClientSideRequest
+ {
+ RequestId = 2,
+ Url = "https://selfservice-torbay.servicebuilder.co.uk/core/addresslookup",
+ Method = "POST",
+ Headers = new()
+ {
+ { "User-Agent", Constants.UserAgent },
+ { "Content-Type", "application/x-www-form-urlencoded" },
+ { "Cookie", requestCookies },
+ { "x-requested-with", "XMLHttpRequest" },
+ },
+ Body = requestBody,
+ };
+
+ var getAddressesResponse = new GetAddressesResponse
+ {
+ NextClientSideRequest = clientSideRequest,
+ };
+
+ return getAddressesResponse;
+ }
+ // Process addresses from response
+ else if (clientSideResponse.RequestId == 2)
+ {
+ using var document = JsonDocument.Parse(clientSideResponse.Content);
+
+ // Iterate through each address, and create a new address object
+ var addresses = new List
();
+ foreach (var element in document.RootElement.EnumerateArray())
+ {
+ var uid = element.GetProperty("Key").GetString();
+ var addressText = element.GetProperty("Value").GetString();
+
+ var address = new Address
+ {
+ Property = addressText?.Trim(),
+ Postcode = postcode,
+ Uid = uid,
+ };
+
+ addresses.Add(address);
+ }
+
+ var getAddressesResponse = new GetAddressesResponse
+ {
+ Addresses = [.. addresses],
+ };
+
+ return getAddressesResponse;
+ }
+
+ // Throw exception for invalid request
+ throw new InvalidOperationException("Invalid client-side request.");
+ }
+
+ ///
+ public GetBinDaysResponse GetBinDays(Address address, ClientSideResponse? clientSideResponse)
+ {
+ // Prepare client-side request for getting form tokens and cookies
+ if (clientSideResponse == null)
+ {
+ var clientSideRequest = new ClientSideRequest
+ {
+ RequestId = 1,
+ Url = "https://selfservice-torbay.servicebuilder.co.uk/renderform?t=62&k=09B72FF904A21A4B01A72AB6CCF28DC95105031C",
+ Method = "GET",
+ Headers = new()
+ {
+ { "User-Agent", Constants.UserAgent },
+ },
+ };
+
+ var getBinDaysResponse = new GetBinDaysResponse
+ {
+ NextClientSideRequest = clientSideRequest,
+ };
+
+ return getBinDaysResponse;
+ }
+ // Prepare client-side request for getting bin days
+ else if (clientSideResponse.RequestId == 1)
+ {
+ var requestCookies = ProcessingUtilities.ParseSetCookieHeaderForRequestCookie(clientSideResponse.Headers["set-cookie"]);
+
+ var token = RequestVerificationTokenRegex().Match(clientSideResponse.Content).Groups["token"].Value;
+ var formGuid = FormGuidRegex().Match(clientSideResponse.Content).Groups["formGuid"].Value;
+ var objectTemplateId = ObjectTemplateIdRegex().Match(clientSideResponse.Content).Groups["objectTemplateId"].Value;
+
+ if (string.IsNullOrWhiteSpace(token) || string.IsNullOrWhiteSpace(formGuid) || string.IsNullOrWhiteSpace(objectTemplateId))
+ {
+ throw new InvalidOperationException("Failed to extract one or more form tokens from the page. The council website may have changed.");
+ }
+
+ var requestBody = ProcessingUtilities.ConvertDictionaryToFormData(new()
+ {
+ { "__RequestVerificationToken", token },
+ { "FormGuid", formGuid },
+ { "ObjectTemplateID", objectTemplateId },
+ { "Trigger", "submit" },
+ { "CurrentSectionID", "0" },
+ { "TriggerCtl", string.Empty },
+ { "FF1168", address.Uid! },
+ { "FF1168lbltxt", "Please select your address" },
+ { "FF1168-text", address.Postcode! },
+ });
+
+ var clientSideRequest = new ClientSideRequest
+ {
+ RequestId = 2,
+ Url = "https://selfservice-torbay.servicebuilder.co.uk/renderform/Form",
+ Method = "POST",
+ Headers = new()
+ {
+ { "User-Agent", Constants.UserAgent },
+ { "Content-Type", "application/x-www-form-urlencoded" },
+ { "Cookie", requestCookies },
+ { "x-requested-with", "XMLHttpRequest" },
+ },
+ Body = requestBody,
+ };
+
+ var getBinDaysResponse = new GetBinDaysResponse
+ {
+ NextClientSideRequest = clientSideRequest,
+ };
+
+ return getBinDaysResponse;
+ }
+ // Process bin days from response
+ else if (clientSideResponse.RequestId == 2)
+ {
+ // Iterate through each bin day row, and create a new bin day object
+ var binDays = new List();
+ foreach (Match match in BinDayRegex().Matches(clientSideResponse.Content)!)
+ {
+ var dateString = match.Groups["date"].Value.Trim();
+ var service = match.Groups["service"].Value.Trim();
+
+ var date = DateOnly.ParseExact(
+ dateString,
+ "dddd dd MMMM yyyy",
+ CultureInfo.InvariantCulture,
+ DateTimeStyles.None
+ );
+
+ var bins = ProcessingUtilities.GetMatchingBins(_binTypes, service);
+
+ var binDay = new BinDay
+ {
+ Date = date,
+ Address = address,
+ Bins = bins,
+ };
+
+ binDays.Add(binDay);
+ }
+
+ var getBinDaysResponse = new GetBinDaysResponse
+ {
+ BinDays = ProcessingUtilities.ProcessBinDays(binDays),
+ };
+
+ return getBinDaysResponse;
+ }
+
+ // Throw exception for invalid request
+ throw new InvalidOperationException("Invalid client-side request.");
+ }
+}
diff --git a/BinDays.Api.IntegrationTests/Collectors/Councils/TorbayCouncilTests.cs b/BinDays.Api.IntegrationTests/Collectors/Councils/TorbayCouncilTests.cs
new file mode 100644
index 00000000..85f2b392
--- /dev/null
+++ b/BinDays.Api.IntegrationTests/Collectors/Councils/TorbayCouncilTests.cs
@@ -0,0 +1,36 @@
+namespace BinDays.Api.IntegrationTests.Collectors.Councils;
+
+using BinDays.Api.Collectors.Collectors;
+using BinDays.Api.Collectors.Collectors.Councils;
+using BinDays.Api.Collectors.Services;
+using BinDays.Api.IntegrationTests.Helpers;
+using System.Threading.Tasks;
+using Xunit;
+using Xunit.Abstractions;
+
+public class TorbayCouncilTests
+{
+ private readonly IntegrationTestClient _client;
+ private static readonly ICollector _collector = new TorbayCouncil();
+ private readonly CollectorService _collectorService = new([_collector]);
+ private readonly ITestOutputHelper _outputHelper;
+
+ public TorbayCouncilTests(ITestOutputHelper outputHelper)
+ {
+ _outputHelper = outputHelper;
+ _client = new IntegrationTestClient(outputHelper);
+ }
+
+ [Theory]
+ [InlineData("TQ1 1NX")]
+ public async Task GetBinDaysTest(string postcode)
+ {
+ await TestSteps.EndToEnd(
+ _client,
+ _collectorService,
+ _collector,
+ postcode,
+ _outputHelper
+ );
+ }
+}