External-postgres-endpoints

stack-clients/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   stack-client:
-    image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.54.1
+    image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.55.0-external-postgres-SNAPSHOT
     secrets:
       - blazegraph_password
       - postgis_password

stack-clients/pom.xml

@@ -7,7 +7,7 @@
 
     <groupId>com.cmclinnovations</groupId>
     <artifactId>stack-clients</artifactId>
-    <version>1.54.1</version>
+    <version>1.55.0-external-postgres-SNAPSHOT</version>
 
     <name>Stack Clients</name>
     <url>https://theworldavatar.io</url>

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/CityDB.java

@@ -12,6 +12,7 @@
 import com.cmclinnovations.stack.clients.citydb.ImpExpOptions;
 import com.cmclinnovations.stack.clients.geoserver.GeoServerClient;
 import com.cmclinnovations.stack.clients.geoserver.GeoServerVectorSettings;
+import com.cmclinnovations.stack.clients.postgis.Database;
 import com.cmclinnovations.stack.clients.postgis.PostGISClient;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -62,18 +63,20 @@ protected void setPreviousFile(Path previousFilePath) {
 
     @Override
     void loadInternal(Dataset parent) {
-        String database = parent.getDatabase();
+        Database database = parent.getDatabase();
+        database.ensureDefault();
+        String databaseName = database.getDatabaseName();
 
         super.loadInternal(parent);
 
         if (null != previousFile) {
             logger.info("Exporting data...");
-            writeOutPrevious(database);
+            writeOutPrevious(databaseName);
         }
 
         if (createTile) {
             logger.info("Creating 3D tiles...");
-            createLayer(database);
+            createLayer(databaseName);
         }
 
     }
@@ -140,7 +143,7 @@ public String getSchema() {
     }
 
     @Override
-    public void createLayers(String workspaceName, String database) {
+    public void createLayers(String workspaceName, Database database) {
         logger.info("Publishing to geoserver...");
         GeoServerClient.getInstance()
                 .createPostGISLayer(workspaceName, database, getSchema(), getTable(), geoServerSettings);

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/DCATUpdateQuery.java

@@ -22,7 +22,7 @@
 import com.cmclinnovations.stack.clients.blazegraph.BlazegraphClient;
 import com.cmclinnovations.stack.clients.core.StackClient;
 import com.cmclinnovations.stack.clients.ontop.OntopClient;
-import com.cmclinnovations.stack.clients.postgis.PostGISClient;
+import com.cmclinnovations.stack.clients.postgis.Database;
 import com.cmclinnovations.stack.clients.rdf4j.Rdf4jClient;
 
 final class DCATUpdateQuery {
@@ -213,10 +213,10 @@ private void addBlazegraphServer(Dataset dataset) {
 
     private void addPostGISServer(Dataset dataset) {
         boolean used = dataset.usesPostGIS();
-        String database = dataset.getDatabase();
-        String url = used ? PostGISClient.getInstance().readEndpointConfig().getJdbcURL(database) : null;
+        Database database = dataset.getDatabase();
+        String url = used ? database.getEndpointConfig().getJdbcURL(database.getDatabaseName()) : null;
 
-        addService(postgisServiceVar, database, SparqlConstants.POSTGIS_SERVICE, url, null, null, used);
+        addService(postgisServiceVar, database.getDatabaseName(), SparqlConstants.POSTGIS_SERVICE, url, null, null, used);
     }
 
     private void addGeoServerServer(Dataset dataset) {

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/Dataset.java

@@ -19,6 +19,7 @@
 import com.cmclinnovations.stack.clients.core.EndpointNames;
 import com.cmclinnovations.stack.clients.geoserver.GeoServerStyle;
 import com.cmclinnovations.stack.clients.geoserver.StaticGeoServerData;
+import com.cmclinnovations.stack.clients.postgis.Database;
 import com.fasterxml.jackson.annotation.JacksonInject;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -40,7 +41,7 @@ public class Dataset extends AbstractDataObject {
     private final Optional<Path> datasetDirectory;
 
     @JsonProperty
-    private final Optional<String> database;
+    private final Optional<Database> database;
     @JsonProperty
     private final Optional<Namespace> namespace;
     @JsonProperty("workspace")
@@ -99,7 +100,7 @@ public class Dataset extends AbstractDataObject {
     Dataset(String name,
             Optional<String> description,
             Optional<Path> datasetDirectory,
-            Optional<String> database,
+            Optional<Database> database,
             Optional<Namespace> namespace,
             Optional<String> workspaceName,
             Optional<List<String>> externalDatasetNames,
@@ -140,12 +141,12 @@ public Path getDirectory() {
         return Path.of("/inputs", "data").resolve(datasetDirectory.orElse(Path.of(name)));
     }
 
-    public String getDatabase() {
+    public Database getDatabase() {
         if (database.isPresent()) {
             LOGGER.warn(NAME_DEPRECATION_NOTICE, "database", getName(), database.get());
             return database.get();
         }
-        return getName();
+        return new Database(getName());
     }
 
     public String getNamespace() {
@@ -195,7 +196,7 @@ public List<String> getOntopMappings() {
     public List<String> getRules() {
         return rules.orElse(Collections.emptyList());
     }
-    
+
     public List<String> getOntopLenses() {
         return ontopLenses.orElse(Collections.emptyList());
     }

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/DatasetLoader.java

@@ -23,6 +23,7 @@
 import com.cmclinnovations.stack.clients.geoserver.GeoServerClient;
 import com.cmclinnovations.stack.clients.geoserver.StaticGeoServerData;
 import com.cmclinnovations.stack.clients.ontop.OntopClient;
+import com.cmclinnovations.stack.clients.postgis.Database;
 import com.cmclinnovations.stack.clients.postgis.PostGISClient;
 import com.cmclinnovations.stack.clients.rdf4j.Rdf4jClient;
 import com.cmclinnovations.stack.clients.utils.JsonHelper;
@@ -207,11 +208,14 @@ private List<ServiceDescription> getServiceDescriptions(String datasetName) {
 
     private void configurePostgres(Dataset dataset, List<DataSubset> dataSubsets) {
         if (dataset.usesPostGIS()) {
+            Database database = dataset.getDatabase();
+            database.ensureDefault();
+            String databaseName = database.getDatabaseName();
             PostGISClient postGISClient = PostGISClient.getInstance();
-            postGISClient.createDatabase(dataset.getDatabase());
+            postGISClient.createDatabase(databaseName);
             dataSubsets.stream().filter(DataSubset::usesPostGIS)
                     .filter(PostgresDataSubset.class::isInstance)
-                    .forEach(subset -> postGISClient.createSchema(dataset.getDatabase(),
+                    .forEach(subset -> postGISClient.createSchema(databaseName,
                             ((PostgresDataSubset) subset).getSchema()));
         }
     }
@@ -263,7 +267,10 @@ private void configureOntop(Dataset dataset, Path directory, List<String> ontolo
             ServiceConfig newOntopServiceConfig = serviceManager.duplicateServiceConfig(EndpointNames.ONTOP,
                     newOntopServiceName);
 
-            newOntopServiceConfig.setEnvironmentVariable(OntopService.ONTOP_DB_NAME, dataset.getDatabase());
+            Database database = dataset.getDatabase();
+            newOntopServiceConfig.setEnvironmentVariable(OntopService.ONTOP_DB_NAME, database.getDatabaseName());
+            newOntopServiceConfig.getContainerSpec().getConfigs().stream().findFirst()
+                    .ifPresent(config -> config.withConfigName(database.getEndpointName()));
             newOntopServiceConfig.getEndpoints()
                     .replaceAll((endpointName, connection) -> new Connection(
                             connection.getUrl(),
@@ -278,16 +285,17 @@ private void configureOntop(Dataset dataset, Path directory, List<String> ontolo
             OntopClient ontopClient = OntopClient.getInstance(newOntopServiceName);
             ontopMappings.forEach(mapping -> ontopClient.updateOBDA(directory.resolve(mapping)));
 
-            if (PostGISClient.DEFAULT_DATABASE_NAME.equals(dataset.getDatabase())) {
+            if (PostGISClient.DEFAULT_DATABASE_NAME.equals(database.getDatabaseName())) {
                 OntopClient defaultOntopClient = OntopClient.getInstance(EndpointNames.ONTOP);
                 ontopMappings.forEach(mapping -> defaultOntopClient.updateOBDA(directory.resolve(mapping)));
             }
 
             ontopClient.uploadOntology(catalogNamespace, ontologyDatasetNames);
 
             ontopClient.uploadRules(dataset.getRules().stream().map(directory::resolve).collect(Collectors.toList()));
-
-            ontopClient.uploadLenses(dataset.getOntopLenses().stream().map(directory::resolve).collect(Collectors.toList()));
+
+            ontopClient.uploadLenses(
+                    dataset.getOntopLenses().stream().map(directory::resolve).collect(Collectors.toList()));
         }
     }
 }

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/DatasetRemover.java

@@ -6,6 +6,7 @@
 import com.cmclinnovations.stack.clients.blazegraph.BlazegraphClient;
 import com.cmclinnovations.stack.clients.core.StackClient;
 import com.cmclinnovations.stack.clients.geoserver.GeoServerClient;
+import com.cmclinnovations.stack.clients.postgis.Database;
 import com.cmclinnovations.stack.clients.postgis.PostGISClient;
 import com.cmclinnovations.stack.services.ServiceManager;
 
@@ -39,6 +40,8 @@ public void removeDataset(Dataset dataset) {
 
             serviceManager.removeService(StackClient.getStackName(), ontopServiceName);
 
+            Database database = dataset.getDatabase();
+            database.ensureDefault();
             GeoServerClient geoServerClient = GeoServerClient.getInstance();
             String workspaceName = dataset.getWorkspaceName();
             // Ensure GeoServer workspace is removed
@@ -48,7 +51,7 @@ public void removeDataset(Dataset dataset) {
             BlazegraphClient.getInstance().removeNamespace(dataset.getNamespace());
 
             // Ensure PostGIS database is removed, if specified
-            PostGISClient.getInstance().removeDatabase(dataset.getDatabase());
+            PostGISClient.getInstance().removeDatabase(database.getDatabaseName());
 
             // Upload styles to GeoServer
             // dataset.getGeoserverStyles().forEach(style ->

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/GeoServerDataSubset.java

@@ -1,5 +1,7 @@
 package com.cmclinnovations.stack.clients.core.datasets;
 
+import com.cmclinnovations.stack.clients.postgis.Database;
+
 public abstract class GeoServerDataSubset extends PostgresDataSubset {
 
     @Override
@@ -13,6 +15,6 @@ void loadInternal(Dataset parent) {
         createLayers(parent.getWorkspaceName(), parent.getDatabase());
     }
 
-    public abstract void createLayers(String workspaceName, String database);
+    public abstract void createLayers(String workspaceName, Database database);
 
 }

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/OSMRouting.java

@@ -5,6 +5,7 @@
 import com.cmclinnovations.stack.clients.core.Options;
 import com.cmclinnovations.stack.clients.geoserver.GeoServerClient;
 import com.cmclinnovations.stack.clients.geoserver.GeoServerVectorSettings;
+import com.cmclinnovations.stack.clients.postgis.Database;
 import com.cmclinnovations.stack.clients.postgis.PGRoutingClient;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
@@ -26,13 +27,13 @@ public void loadData(Path dirPath, String database, String baseIRI) {
     }
 
     @Override
-    public void createLayers(String workspaceName, String database) {
+    public void createLayers(String workspaceName, Database database) {
         createLayer(workspaceName, database, prefixTableName("ways"), waysGeoServerSettings);
         createLayer(workspaceName, database, prefixTableName("ways_vertices_pgr"), verticesGeoServerSettings);
         createLayer(workspaceName, database, prefixTableName("pointsofinterest"), poiGeoServerSettings);
     }
 
-    private void createLayer(String workspaceName, String database, String layerName,
+    private void createLayer(String workspaceName, Database database, String layerName,
             GeoServerVectorSettings geoServerVectorSettings) {
         GeoServerClient.getInstance()
                 .createPostGISLayer(workspaceName, database, getSchema(), layerName, geoServerVectorSettings);

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/PostgresDataSubset.java

@@ -7,6 +7,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.cmclinnovations.stack.clients.postgis.Database;
 import com.cmclinnovations.stack.clients.postgis.PostGISClient;
 import com.cmclinnovations.stack.clients.utils.JsonHelper;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -49,17 +50,19 @@ public boolean usesPostGIS() {
 
     @Override
     void loadInternal(Dataset parent) {
-        String database = parent.getDatabase();
+        Database database = parent.getDatabase();
+        database.ensureDefault();
+        String databaseName = database.getDatabaseName();
         Optional<Path> subdirectory = this.getSubdirectory();
         if (subdirectory.isPresent()) {
             Path dataSubsetDirectory = parent.getDirectory().resolve(subdirectory.get());
-            PostGISClient.getInstance().resetSchema(database);
-            loadData(dataSubsetDirectory, database, parent.baseIRI());
+            PostGISClient.getInstance().resetSchema(databaseName);
+            loadData(dataSubsetDirectory, databaseName, parent.baseIRI());
         } else {
             logger.warn("No Subdirectory specified, Continuing with SQL process and creation without data upload");
         }
-        runSQLPostProcess(database);
-        PostGISClient.getInstance().resetSchema(database);
+        runSQLPostProcess(databaseName);
+        PostGISClient.getInstance().resetSchema(databaseName);
     }
 
     public abstract void loadData(Path dataSubsetDir, String database, String baseIRI);

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/RML.java

@@ -1,7 +1,6 @@
 package com.cmclinnovations.stack.clients.core.datasets;
 
 import java.nio.file.Path;
-import java.util.Map;
 
 import com.cmclinnovations.stack.clients.rml.RmlMapperClient;
 

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/Raster.java

@@ -9,6 +9,7 @@
 import com.cmclinnovations.stack.clients.geoserver.GeoServerClient;
 import com.cmclinnovations.stack.clients.geoserver.GeoServerRasterSettings;
 import com.cmclinnovations.stack.clients.geoserver.MultidimSettings;
+import com.cmclinnovations.stack.clients.postgis.Database;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 public class Raster extends GeoServerDataSubset {
@@ -30,7 +31,7 @@ public void loadData(Path dirPath, String database, String baseIRI) {
     }
 
     @Override
-    public void createLayers(String workspaceName, String database) {
+    public void createLayers(String workspaceName, Database database) {
         GeoServerClient.getInstance()
                 .createGeoTiffLayer(workspaceName, getTable(), database, getSchema(),
                         geoServerSettings, mdimSettings);

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/Vector.java

@@ -6,6 +6,7 @@
 import com.cmclinnovations.stack.clients.gdal.Ogr2OgrOptions;
 import com.cmclinnovations.stack.clients.geoserver.GeoServerClient;
 import com.cmclinnovations.stack.clients.geoserver.GeoServerVectorSettings;
+import com.cmclinnovations.stack.clients.postgis.Database;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 public class Vector extends GeoServerDataSubset {
@@ -24,7 +25,7 @@ public void loadData(Path dirPath, String database, String baseIRI) {
     }
 
     @Override
-    public void createLayers(String workspaceName, String database) {
+    public void createLayers(String workspaceName, Database database) {
         GeoServerClient.getInstance()
                 .createPostGISLayer(workspaceName, database, getSchema(), getTable(), geoServerSettings);
     }

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/XtoCityDB.java

@@ -34,12 +34,12 @@ protected void loadDataInternal(Path dataSubsetDir, String database, String base
         GDALClient.getInstance()
                 .uploadVectorFilesToPostGIS(database, getSchema(), getTable(), dataSubsetDir.toString(),
                         ogr2ogrOptions, false);
-        CityDBClient.getInstance()
-                .updateDatabase(database, getSridIn());
-        CityDBClient.getInstance().preparePGforCityDB(database, getTable(), JsonHelper.handleFileValues(preprocessSql),
-                minArea, columnMap);
-        CityDBClient.getInstance().populateCityDBbySQL(database, lineage, columnMap);
-        CityDBClient.getInstance().addIRIs(database, baseIRI);
+        CityDBClient instance = CityDBClient.getInstance();
+        instance.updateDatabase(database, getSridIn());
+        instance.preparePGforCityDB(database, getTable(), JsonHelper.handleFileValues(preprocessSql), minArea,
+                columnMap);
+        instance.populateCityDBbySQL(database, lineage, columnMap);
+        instance.addIRIs(database, baseIRI);
     }
 
     @Override