Add Traefik support

stack-clients/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   stack-client:
-    image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.56.2
+    image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.57.0-traefik-support-SNAPSHOT
     secrets:
       - blazegraph_password
       - postgis_password

stack-clients/pom.xml

@@ -7,15 +7,15 @@
 
     <groupId>com.cmclinnovations</groupId>
     <artifactId>stack-clients</artifactId>
-    <version>1.56.2</version>
+    <version>1.57.0-traefik-support-SNAPSHOT</version>
 
     <name>Stack Clients</name>
     <url>https://theworldavatar.io</url>
 
     <parent>
         <groupId>uk.ac.cam.cares.jps</groupId>
         <artifactId>jps-parent-pom</artifactId>
-        <version>2.3.2</version>
+        <version>2.4.0</version>
     </parent>
 
     <properties>

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/StackClient.java

@@ -36,6 +36,7 @@ public final class StackClient {
     private static StackHost stackHost = new StackHost();
 
     private static boolean isolated = false;
+    private static String reverseProxyName;
 
     static {
         String envVarStackName = System.getenv(StackClient.STACK_NAME_KEY);
@@ -107,6 +108,14 @@ public static Path getAbsDataPath() {
         return getStackBaseDir().resolve("inputs").resolve("data");
     }
 
+    public static void setReverseProxyName(String reverseProxyName) {
+        StackClient.reverseProxyName = reverseProxyName;
+    }
+
+    public static String getReverseProxyName() {
+        return reverseProxyName;
+    }
+
     /**
      * Get a RemoteRDBStoreClient for the named Postgres RDB running in this stack.
      *

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/RML.java

@@ -1,7 +1,6 @@
 package com.cmclinnovations.stack.clients.core.datasets;
 
 import java.nio.file.Path;
-import java.util.Map;
 
 import com.cmclinnovations.stack.clients.rml.RmlMapperClient;
 

stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java

@@ -53,7 +53,9 @@
 import com.github.dockerjava.core.DefaultDockerClientConfig.Builder;
 import com.github.dockerjava.core.DockerClientBuilder;
 import com.github.dockerjava.core.DockerClientConfig;
-import com.github.dockerjava.core.command.ExecStartResultCallback;
+import com.github.dockerjava.api.async.ResultCallback;
+import com.github.dockerjava.api.model.Frame;
+import com.github.dockerjava.api.model.StreamType;
 import com.github.dockerjava.httpclient5.ApacheDockerHttpClient;
 import com.github.dockerjava.transport.DockerHttpClient;
 
@@ -115,7 +117,6 @@ public String executeSimpleCommand(String containerId, String... cmd) {
                 .withOutputStream(outputStream)
                 .withErrorStream(outputStream)
                 .exec();
-        String output = outputStream.toString();
         return execId;
     }
 
@@ -231,11 +232,21 @@ public String exec() {
                     execStartCmd.withStdIn(inputStream);
                 }
 
-                // ExecStartResultCallback is marked deprecated but seems to do exactly what we
-                // want and without knowing why it is deprecated any issues with it can't be
-                // overcome anyway.
-                try (ExecStartResultCallback result = execStartCmd
-                        .exec(new ExecStartResultCallback(outputStream, errorStream))) {
+                try (ResultCallback.Adapter<Frame> result = execStartCmd
+                        .exec(new ResultCallback.Adapter<Frame>() {
+                            @Override
+                            public void onNext(Frame frame) {
+                                try {
+                                    if (frame.getStreamType() == StreamType.STDOUT && outputStream != null) {
+                                        outputStream.write(frame.getPayload());
+                                    } else if (frame.getStreamType() == StreamType.STDERR && errorStream != null) {
+                                        errorStream.write(frame.getPayload());
+                                    }
+                                } catch (IOException ex) {
+                                    throw new RuntimeException("Failed to write frame payload", ex);
+                                }
+                            }
+                        })) {
                     if (wait) {
                         if (!result.awaitCompletion(evaluationTimeout, TimeUnit.SECONDS)) {
                             LOGGER.warn("Docker exec command '{}' still running after the {} second execution timeout.",
@@ -553,7 +564,7 @@ public boolean isContainerUp(String containerName) {
 
     public String getContainerId(String containerName) {
         return getContainer(containerName).map(Container::getId)
-                .orElseThrow(() -> new NoSuchElementException("Cannot get container "+containerName+"."));
+                .orElseThrow(() -> new NoSuchElementException("Cannot get container " + containerName + "."));
     }
 
     private Map<String, List<String>> convertToConfigFilterMap(String configName, Map<String, String> labelMap) {

stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/PodmanClient.java

@@ -7,15 +7,16 @@
 
 import com.cmclinnovations.stack.clients.core.StackClient;
 import com.cmclinnovations.stack.clients.utils.JsonHelper;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.dockerjava.api.model.Config;
+import com.github.dockerjava.jaxrs.ApiClientExtension;
+
 import io.theworldavatar.swagger.podman.ApiClient;
 import io.theworldavatar.swagger.podman.ApiException;
 import io.theworldavatar.swagger.podman.api.NetworksApi;
 import io.theworldavatar.swagger.podman.api.SecretsApi;
 import io.theworldavatar.swagger.podman.model.Network;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.github.dockerjava.api.model.Config;
-import com.github.dockerjava.jaxrs.ApiClientExtension;
 
 public class PodmanClient extends DockerClient {
 

stack-clients/src/main/java/com/cmclinnovations/stack/clients/rdf4j/Rdf4jClient.java

@@ -4,7 +4,6 @@
 
 import org.eclipse.rdf4j.federated.repository.FedXRepositoryConfigBuilder;
 import org.eclipse.rdf4j.repository.config.RepositoryConfig;
-import org.eclipse.rdf4j.repository.config.RepositoryImplConfig;
 import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager;
 import org.eclipse.rdf4j.repository.sail.config.SailRepositoryConfig;
 import org.eclipse.rdf4j.repository.sparql.config.SPARQLRepositoryConfig;

stack-clients/src/main/java/com/cmclinnovations/stack/services/ServiceManager.java

@@ -167,15 +167,17 @@ public <S extends Service> S initialiseService(String stackName, String serviceN
             DockerService dockerService = getOrInitialiseService(stackName, StackClient.getContainerEngineName());
             dockerService.doPreStartUpConfiguration(newContainerService);
 
+            if (!StackClient.getReverseProxyName().equals(serviceName)) {
+                ReverseProxyService reverseProxyService = getOrInitialiseService(stackName,
+                        StackClient.getReverseProxyName());
+                reverseProxyService.addService(newContainerService);
+            }
+
             dockerService.writeEndpointConfigs(newContainerService);
             if (dockerService.startContainer(newContainerService)) {
                 dockerService.doFirstTimePostStartUpConfiguration(newContainerService);
             }
             dockerService.doEveryTimePostStartUpConfiguration(newContainerService);
-            if (!NginxService.TYPE.equals(serviceName)) {
-                ReverseProxyService reverseProxyService = getOrInitialiseService(stackName, NginxService.TYPE);
-                reverseProxyService.addService(newContainerService);
-            }
         }
 
         services.put(serviceName, newService);

stack-clients/src/main/java/com/cmclinnovations/stack/services/TraefikService.java

@@ -0,0 +1,89 @@
+package com.cmclinnovations.stack.services;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.cmclinnovations.stack.clients.core.StackClient;
+import com.cmclinnovations.stack.clients.utils.FileUtils;
+import com.cmclinnovations.stack.services.config.ServiceConfig;
+import com.github.dockerjava.api.model.ContainerSpec;
+
+public class TraefikService extends ContainerService implements ReverseProxyService {
+
+    public static final String TYPE = "traefik";
+
+    private static final String TRAEFIK_CONF_DIR = "/etc/traefik/";
+    private static final String TRAEFIK_CONFIG_TEMPLATE = "traefik/configs/traefik.yml";
+
+    public TraefikService(String stackName, ServiceConfig config) {
+        super(stackName, config);
+    }
+
+    @Override
+    public void doFirstTimePostStartUpConfiguration() {
+        try (InputStream inStream = new BufferedInputStream(
+                TraefikService.class.getResourceAsStream(TRAEFIK_CONFIG_TEMPLATE))) {
+
+            String configContent = new String(inStream.readAllBytes(), StandardCharsets.UTF_8);
+            // Replace the ${STACK_NAME} placeholder with actual stack name
+            String stackName = getEnvironmentVariable(StackClient.STACK_NAME_KEY);
+            configContent = configContent.replace("${STACK_NAME}", stackName);
+
+            // Write the configuration file to the container
+            Map<String, byte[]> files = new HashMap<>();
+            files.put("traefik.yml", configContent.getBytes(StandardCharsets.UTF_8));
+            sendFilesContent(files, TRAEFIK_CONF_DIR);
+
+        } catch (IOException ex) {
+            throw new RuntimeException("Failed to configure Traefik", ex);
+        }
+    }
+
+    @Override
+    public void addService(ContainerService service) {
+        ContainerSpec containerSpec = service.getContainerSpec();
+        Map<String, String> labels = new HashMap<>();
+        containerSpec.withLabels(labels);
+        labels.put("traefik.enable", "true");
+
+        service.getConfig().getEndpoints().forEach((endpointName, connection) -> {
+            URI externalPath = connection.getExternalPath();
+            if (null != externalPath) {
+                String serviceName = service.getName();
+                String routerName = serviceName + "_" + endpointName;
+                String pathPrefix = FileUtils.fixSlashes(externalPath.getPath(), true, false);
+
+                // Configure router with path prefix rule
+                labels.put("traefik.http.routers." + routerName + ".rule",
+                        "PathPrefix(`" + pathPrefix + "`)");
+                labels.put("traefik.http.routers." + routerName + ".entrypoints", "web");
+
+                // Configure service with the internal port
+                URL url = connection.getUrl();
+                int port = url.getPort();
+                if (port == -1) {
+                    port = 80; // Default port
+                }
+                labels.put("traefik.http.routers." + routerName + ".service", routerName);
+                labels.put("traefik.http.services." + routerName + ".loadbalancer.server.port",
+                        String.valueOf(port));
+            }
+
+            // TODO: Set the labels correctly
+            // labels.put("traefik.http.routers." + service.getContainerName() + ".rule",
+            // "PathPrefix(`" + FileUtils.fixSlashes(connection.getExternalPath().getPath(),
+            // true, false) + "`)");
+            // labels.put(
+            // "traefik.http.services." + service.getServiceName() +
+            // ".loadbalancer.server.port",
+            // String.valueOf(connection.getInternalPort()));
+        });
+    }
+
+}

stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/traefik.json

@@ -0,0 +1,40 @@
+{
+    "type": "traefik",
+    "ServiceSpec": {
+        "Name": "traefik",
+        "TaskTemplate": {
+            "ContainerSpec": {
+                "Image": "traefik:v3.6",
+                "Mounts": [
+                    {
+                        "Type": "volume",
+                        "Source": "traefik_config",
+                        "Target": "/etc/traefik"
+                    }
+                ]
+            }
+        },
+        "EndpointSpec": {
+            "Ports": [
+                {
+                    "Name": "web",
+                    "Protocol": "tcp",
+                    "TargetPort": "80",
+                    "PublishedPort": "3838"
+                },
+                {
+                    "Name": "websecure",
+                    "Protocol": "tcp",
+                    "TargetPort": "443",
+                    "PublishedPort": "443"
+                },
+                {
+                    "Name": "dashboard",
+                    "Protocol": "tcp",
+                    "TargetPort": "8080",
+                    "PublishedPort": "8080"
+                }
+            ]
+        }
+    }
+}

stack-clients/src/main/resources/com/cmclinnovations/stack/services/traefik/configs/traefik.yml

@@ -0,0 +1,21 @@
+api:
+  dashboard: true
+  insecure: true
+
+entryPoints:
+  web:
+    address: ":80"
+  websecure:
+    address: ":443"
+  traefik:
+    address: ":8080"
+
+providers:
+  docker:
+    endpoint: "unix:///var/run/docker.sock"
+    exposedByDefault: false
+    swarmMode: true
+    network: "${STACK_NAME}"
+
+log:
+  level: INFO

stack-data-uploader/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   stack-data-uploader:
-    image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.56.2
+    image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.57.0-traefik-support-SNAPSHOT
     secrets:
       - blazegraph_password
       - postgis_password

stack-data-uploader/pom.xml

@@ -7,7 +7,7 @@
 
     <groupId>com.cmclinnovations</groupId>
     <artifactId>stack-data-uploader</artifactId>
-    <version>1.56.2</version>
+    <version>1.57.0-traefik-support-SNAPSHOT</version>
 
     <name>Stack Data Uploader</name>
     <url>https://theworldavatar.io</url>
@@ -38,7 +38,7 @@
         <dependency>
             <groupId>com.cmclinnovations</groupId>
             <artifactId>stack-clients</artifactId>
-            <version>1.56.2</version>
+            <version>1.57.0-traefik-support-SNAPSHOT</version>
         </dependency>
 
         <dependency>

stack-manager/.vscode/tasks.json

@@ -33,7 +33,10 @@
             ],
             "options": {
                 "shell": {
-                    "executable": "bash"
+                    "executable": "bash",
+                    "args": [
+                        "-c"
+                    ]
                 }
             }
         },

stack-manager/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   stack-manager:
-    image: ghcr.io/theworldavatar/stack-manager${IMAGE_SUFFIX}:1.56.2
+    image: ghcr.io/theworldavatar/stack-manager${IMAGE_SUFFIX}:1.57.0-traefik-support-SNAPSHOT
     environment:
       EXTERNAL_PORT: "${EXTERNAL_PORT-3838}"
       STACK_BASE_DIR: "${STACK_BASE_DIR}"

stack-manager/pom.xml

@@ -7,7 +7,7 @@
 
     <groupId>com.cmclinnovations</groupId>
     <artifactId>stack-manager</artifactId>
-    <version>1.56.2</version>
+    <version>1.57.0-traefik-support-SNAPSHOT</version>
 
     <name>Stack Manager</name>
     <url>https://theworldavatar.io</url>
@@ -38,7 +38,7 @@
         <dependency>
             <groupId>com.cmclinnovations</groupId>
             <artifactId>stack-clients</artifactId>
-            <version>1.56.2</version>
+            <version>1.57.0-traefik-support-SNAPSHOT</version>
         </dependency>
 
         <dependency>

stack-manager/src/main/java/com/cmclinnovations/stack/Stack.java

@@ -83,6 +83,7 @@ private Stack(String name, ServiceManager manager, StackConfig config) {
         if (null != config) {
             StackClient.setStackHost(config.getHost());
             StackClient.setIsolated(config.isIsolated());
+            StackClient.setReverseProxyName(config.getReverseProxyName());
         }
     }