Add cache url and fix issue on 403 due to server only allow browser b…

server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/WfsWmsConfig.java

@@ -5,23 +5,38 @@
 import au.org.aodn.ogcapi.server.core.service.wfs.WfsServer;
 import au.org.aodn.ogcapi.server.core.service.wms.WmsServer;
 import au.org.aodn.ogcapi.server.core.util.RestTemplateUtils;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
 import org.springframework.web.client.RestTemplate;
 
 @Configuration
 public class WfsWmsConfig {
 
+    @ConditionalOnMissingBean(name="pretendUserEntity")
+    @Bean("pretendUserEntity")
+    public HttpEntity<?> createPretendUserEntity() {
+        // Some server do not allow program to scrap the content, so we need to pretend to be a client
+        HttpHeaders headers = new HttpHeaders();
+        headers.set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36");
+
+        return new HttpEntity<>(headers);
+    }
+
     @Bean
     public WfsServer createWfsServer(Search search,
                                      DownloadableFieldsService downloadableFieldsService,
                                      RestTemplate restTemplate,
-                                     RestTemplateUtils restTemplateUtils) {
-        return new WfsServer(search, downloadableFieldsService, restTemplate, restTemplateUtils);
+                                     RestTemplateUtils restTemplateUtils,
+                                     @Qualifier("pretendUserEntity") HttpEntity<?> entity) {
+        return new WfsServer(search, downloadableFieldsService, restTemplate, restTemplateUtils, entity);
     }
 
     @Bean
-    public WmsServer createWmsServer() {
-        return new WmsServer();
+    public WmsServer createWmsServer(@Qualifier("pretendUserEntity") HttpEntity<?> entity) {
+        return new WmsServer(entity);
     }
 }

server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheWarm.java

@@ -7,6 +7,8 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.cache.annotation.CacheEvict;
 import org.springframework.context.annotation.Lazy;
+import org.springframework.retry.annotation.Backoff;
+import org.springframework.retry.annotation.Retryable;
 import org.springframework.scheduling.annotation.Scheduled;
 import java.util.List;
 import java.util.Map;
@@ -17,15 +19,18 @@
  */
 @Slf4j
 public class CacheWarm {
-    // Hardcode server list as not expect to change much overtime, add more if needed
+    // Hardcode server list as not expect to change much overtime, add more if needed, the operation is
+    // heavy so we warm the cache when start
     protected List<String> getCapabilitiesUrls = List.of(
             "https://data.aad.gov.au/geoserver/underway/wms",
             "https://www.cmar.csiro.au/geoserver/caab/wms",
             "https://www.cmar.csiro.au/geoserver/ereefs/wms",
             "https://www.cmar.csiro.au/geoserver/ea-be/wms",
             "https://www.cmar.csiro.au/geoserver/gsfm/wms",
             "https://www.cmar.csiro.au/geoserver/nerp/wms",
-            "https://www.cmar.csiro.au/geoserver/AusSeabed/wms"
+            "https://www.cmar.csiro.au/geoserver/AusSeabed/wms",
+            "https://geoserver.apps.aims.gov.au/aims/wms",
+            "https://geoserver.apps.aims.gov.au/reefcloud/wms"
     );
     protected WmsServer wmsServer;
     protected GeometryUtils geometryUtils;
@@ -73,6 +78,7 @@ protected void evictGetCapabilities(String key){
         );
     }
 
+    @Retryable(maxAttempts = 3, backoff = @Backoff(delay = 1000))
     protected void warmGetCapabilities(String url) {
         try {
             // Call and warm cache

server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java

@@ -16,6 +16,8 @@
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.cache.annotation.Cacheable;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpMethod;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.client.RestClientException;
 import org.springframework.web.client.RestTemplate;
@@ -39,11 +41,13 @@ public class WfsServer {
     protected RestTemplateUtils restTemplateUtils;
     protected RestTemplate restTemplate;
     protected Search search;
+    protected HttpEntity<?> pretendUserEntity;
 
     public WfsServer(Search search,
                      DownloadableFieldsService downloadableFieldsService,
                      RestTemplate restTemplate,
-                     RestTemplateUtils restTemplateUtils) {
+                     RestTemplateUtils restTemplateUtils,
+                     HttpEntity<?> entity) {
 
         xmlMapper = new XmlMapper();
         xmlMapper.registerModule(new JavaTimeModule()); // Add JavaTimeModule
@@ -53,6 +57,7 @@ public WfsServer(Search search,
         this.restTemplate = restTemplate;
         this.restTemplateUtils = restTemplateUtils;
         this.downloadableFieldsService = downloadableFieldsService;
+        this.pretendUserEntity = entity;
     }
 
     /**
@@ -77,7 +82,12 @@ public List<DownloadableFieldModel> getDownloadableFields(String collectionId, F
                 try {
                     if (uri != null) {
                         log.debug("Try Url to wfs {}", uri);
-                        ResponseEntity<String> response = restTemplateUtils.handleRedirect(uri, restTemplate.getForEntity(uri, String.class), String.class);
+                        ResponseEntity<String> response = restTemplateUtils.handleRedirect(
+                                uri,
+                                restTemplate.exchange(uri, HttpMethod.GET, pretendUserEntity, String.class),
+                                String.class,
+                                pretendUserEntity
+                        );
 
                         if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) {
                             return DownloadableFieldsService.convertWfsResponseToDownloadableFields(

server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java

@@ -21,8 +21,7 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.cache.annotation.Cacheable;
 import org.springframework.context.annotation.Lazy;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
+import org.springframework.http.*;
 import org.springframework.web.client.RestClientException;
 import org.springframework.web.client.RestTemplate;
 import org.springframework.web.util.UriComponents;
@@ -61,10 +60,14 @@ public class WmsServer {
     @Autowired
     protected WmsServer self;
 
-    public WmsServer() {
+    protected final HttpEntity<?> pretendUserEntity;
+
+    public WmsServer(HttpEntity<?> entity) {
         xmlMapper = new XmlMapper();
         xmlMapper.registerModule(new JavaTimeModule()); // Add JavaTimeModule
         xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+        pretendUserEntity = entity;
     }
     /**
      * This function is used to append the CQL filter to the geonetwork query, it will guess the correct dataTime field by
@@ -425,7 +428,7 @@ public FeatureInfoResponse getMapFeatures(String collectionId, FeatureRequest re
             List<String> urls = createMapFeatureQueryUrl(mapServerUrl.get(), collectionId, request);
             // Try one by one, we exit when any works
             for (String url : urls) {
-                ResponseEntity<String> response = restTemplateUtils.handleRedirect(url, restTemplate.getForEntity(url, String.class, Collections.emptyMap()), String.class);
+                ResponseEntity<String> response = restTemplateUtils.handleRedirect(url, restTemplate.getForEntity(url, String.class, Collections.emptyMap()), String.class, pretendUserEntity);
                 if (response.getStatusCode().is2xxSuccessful()) {
                     // Now try to unify the return
                     if (MediaType.TEXT_HTML.isCompatibleWith(response.getHeaders().getContentType())) {
@@ -456,7 +459,7 @@ public DescribeLayerResponse describeLayer(String collectionId, FeatureRequest r
             // Try one by one, we exit when any works
             for (String url : urls) {
                 try {
-                    ResponseEntity<String> response = restTemplateUtils.handleRedirect(url, restTemplate.getForEntity(url, String.class, Collections.emptyMap()), String.class);
+                    ResponseEntity<String> response = restTemplateUtils.handleRedirect(url, restTemplate.exchange(url, HttpMethod.GET, pretendUserEntity, String.class), String.class, pretendUserEntity);
                     if (response.getStatusCode().is2xxSuccessful()) {
                         DescribeLayerResponse layer = xmlMapper.readValue(response.getBody(), DescribeLayerResponse.class);
                         if (layer.getLayerDescription() != null) {
@@ -489,7 +492,7 @@ public byte[] getMapTile(String collectionId, FeatureRequest request) throws URI
             // Try one by one, we exit when any works
             for (String url : urls) {
                 log.debug("map tile request for layer name {} url {} ", request.getLayerName(), url);
-                ResponseEntity<byte[]> response = restTemplateUtils.handleRedirect(url, restTemplate.getForEntity(url, byte[].class, Collections.emptyMap()), byte[].class);
+                ResponseEntity<byte[]> response = restTemplateUtils.handleRedirect(url, restTemplate.exchange(url, HttpMethod.GET, pretendUserEntity, byte[].class), byte[].class, pretendUserEntity);
                 if (response.getStatusCode().is2xxSuccessful()) {
                     return response.getBody();
                 }
@@ -531,7 +534,7 @@ public List<LayerInfo> fetchCapabilitiesLayersByUrl(String wmsServerUrl) {
             // Build GetCapabilities URL
             UriComponentsBuilder builder = UriComponentsBuilder
                     .newInstance()
-                    .scheme(components.getScheme())
+                    .scheme("https")        // hardcode to be https to avoid redirect
                     .port(components.getPort())
                     .host(components.getHost())
                     .path(components.getPath() != null ? components.getPath() : "/geoserver/ows")
@@ -541,12 +544,8 @@ public List<LayerInfo> fetchCapabilitiesLayersByUrl(String wmsServerUrl) {
             String url = builder.build().toUriString();
             log.debug("GetCapabilities URL: {}", url);
 
-            // Make the HTTP call
-            ResponseEntity<String> response = restTemplateUtils.handleRedirect(
-                    url,
-                    restTemplate.getForEntity(url, String.class, Collections.emptyMap()),
-                    String.class
-            );
+            // Make the HTTPS call
+            ResponseEntity<String> response = restTemplate.exchange(url, HttpMethod.GET, pretendUserEntity, String.class);
 
             if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) {
                 // Parse XML response
@@ -569,7 +568,7 @@ public List<LayerInfo> fetchCapabilitiesLayersByUrl(String wmsServerUrl) {
                     return layers;
                 }
             }
-        } catch (RestClientException | URISyntaxException | JsonProcessingException e) {
+        } catch (RestClientException | JsonProcessingException e) {
             log.error("Error fetching GetCapabilities for URL: {}", wmsServerUrl, e);
             throw new RuntimeException(e);
         }

server/src/main/java/au/org/aodn/ogcapi/server/core/util/RestTemplateUtils.java

@@ -1,11 +1,12 @@
 package au.org.aodn.ogcapi.server.core.util;
 
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpMethod;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.client.RestTemplate;
 
 import java.net.URISyntaxException;
-import java.util.Collections;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -27,7 +28,7 @@ public RestTemplateUtils(RestTemplate restTemplate) {
      * @param <T> The type of the return type
      * @throws URISyntaxException - Not expect to throw
      */
-    public <T> ResponseEntity<T> handleRedirect(String sourceUrl, ResponseEntity<T> response, Class<T> type) throws URISyntaxException {
+    public <T> ResponseEntity<T> handleRedirect(String sourceUrl, ResponseEntity<T> response, Class<T> type, HttpEntity<?> entity) throws URISyntaxException {
         // Redirect should happen automatically but it does not so here is a safe-guard
         // the reason happens because http is use but redirect to https
         if(response != null && response.getStatusCode().is3xxRedirection() && response.getHeaders().getLocation() != null) {
@@ -44,7 +45,7 @@ public <T> ResponseEntity<T> handleRedirect(String sourceUrl, ResponseEntity<T>
             if(haveSameHost(sourceUrl, redirect)) {
                 // Only allow redirect to same server.
                 log.info("Redirect from {} to {}", sourceUrl , redirect);
-                return restTemplate.getForEntity(redirect, type, Collections.emptyMap());
+                return restTemplate.exchange(redirect, HttpMethod.GET, entity, type);
             }
             else {
                 log.error("Redirect to different host not allowed, from {} to {}", sourceUrl , redirect);

server/src/test/java/au/org/aodn/ogcapi/server/core/configuration/TestConfig.java

@@ -1,10 +1,11 @@
 package au.org.aodn.ogcapi.server.core.configuration;
 
-import au.org.aodn.ogcapi.server.core.service.Search;
 import org.mockito.Mockito;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Primary;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
 import org.springframework.web.client.RestTemplate;
 
 @Configuration
@@ -15,4 +16,14 @@ public class TestConfig {
     public RestTemplate createMockRestTemplate() {
         return Mockito.mock(RestTemplate.class);
     }
+
+    @Bean("pretendUserEntity")
+    @Primary
+    public HttpEntity<?> createPretendUserEntity() {
+        // Satisfy depends
+        HttpHeaders headers = new HttpHeaders();
+        headers.set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36");
+
+        return new HttpEntity<>(headers);
+    }
 }

server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadableFieldsServiceTest.java

@@ -19,9 +19,12 @@
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.boot.autoconfigure.cache.CacheAutoConfiguration;
 import org.springframework.boot.autoconfigure.jackson.JacksonAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpMethod;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.test.context.bean.override.mockito.MockitoBean;
@@ -64,6 +67,10 @@ public class DownloadableFieldsServiceTest {
     @Autowired
     private WfsServer wfsServer;
 
+    @Autowired
+    @Qualifier("pretendUserEntity")
+    private HttpEntity<?> entity;
+
     @MockitoBean
     private DasService dasService;
 
@@ -133,7 +140,7 @@ public void testGetDownloadableFieldsSuccess() {
 
         String id = "id";
 
-        when(restTemplate.getForEntity(any(String.class), eq(String.class)))
+        when(restTemplate.exchange(any(String.class), eq(HttpMethod.GET), eq(entity), eq(String.class)))
                 .thenReturn(new ResponseEntity<>(mockWfsResponse, HttpStatus.OK));
 
         when(search.searchCollections(eq(id)))
@@ -203,7 +210,7 @@ public void testGetDownloadableFieldsEmptyResponse() {
 
         String id = "id2";
 
-        when(restTemplate.getForEntity(any(String.class), eq(String.class)))
+        when(restTemplate.exchange(any(String.class), eq(HttpMethod.GET), eq(entity), eq(String.class)))
                 .thenReturn(new ResponseEntity<>(mockWfsResponse, HttpStatus.NOT_FOUND));
 
         when(search.searchCollections(eq(id)))
@@ -244,7 +251,7 @@ public void testGetDownloadableFieldsWfsError() {
         when(search.searchCollections(eq(id)))
                 .thenReturn(stac);
 
-        when(restTemplate.getForEntity(any(String.class), eq(String.class)))
+        when(restTemplate.exchange(any(String.class), eq(HttpMethod.GET), eq(entity), eq(String.class)))
                 .thenReturn(new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR));
 
         DownloadableFieldsNotFoundException exception = assertThrows(
@@ -280,7 +287,7 @@ public void testGetDownloadableFieldsNetworkError() {
                 .thenReturn(stac);
 
         // Mock network error
-        when(restTemplate.getForEntity(any(String.class), eq(String.class)))
+        when(restTemplate.exchange(any(String.class), eq(HttpMethod.GET), eq(entity), eq(String.class)))
                 .thenThrow(new RuntimeException("Connection timeout"));
 
         RuntimeException exception = assertThrows(