From d096aa48e3bdcfa26adf002536ef26294eec0826 Mon Sep 17 00:00:00 2001 From: Salvatore Mongiardo Date: Thu, 5 Mar 2026 17:01:04 +0100 Subject: [PATCH] [Qdrant] Adding system-x --- bom/pom.xml | 15 +- system-x/services/ai/pom.xml | 1 + system-x/services/ai/qdrant/pom.xml | 18 ++ .../qdrant/resource/local/LocalQdrant.java | 35 ++++ .../resource/local/QdrantContainer.java | 13 ++ .../resource/openshift/OpenshiftQdrant.java | 176 ++++++++++++++++++ .../software/tnb/qdrant/service/Qdrant.java | 32 ++++ .../qdrant/validation/QdrantValidation.java | 51 +++++ system-x/services/all/pom.xml | 12 +- 9 files changed, 344 insertions(+), 9 deletions(-) create mode 100644 system-x/services/ai/qdrant/pom.xml create mode 100644 system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/resource/local/LocalQdrant.java create mode 100644 system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/resource/local/QdrantContainer.java create mode 100644 system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/resource/openshift/OpenshiftQdrant.java create mode 100644 system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/service/Qdrant.java create mode 100644 system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/validation/QdrantValidation.java diff --git a/bom/pom.xml b/bom/pom.xml index 4125caae9..6129328f0 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -194,6 +194,16 @@ system-x-docling 1.0-SNAPSHOT + + software.tnb + system-x-ollama + 1.0-SNAPSHOT + + + software.tnb + system-x-qdrant + 1.0-SNAPSHOT + software.tnb system-x-elasticsearch @@ -419,11 +429,6 @@ system-x-observability 1.0-SNAPSHOT - - software.tnb - system-x-ollama - 1.0-SNAPSHOT - software.tnb system-x-opensearch diff --git a/system-x/services/ai/pom.xml b/system-x/services/ai/pom.xml index 7a7f302d0..9286a71b8 100644 --- a/system-x/services/ai/pom.xml +++ b/system-x/services/ai/pom.xml @@ -18,5 +18,6 @@ docling milvus ollama + qdrant diff --git a/system-x/services/ai/qdrant/pom.xml b/system-x/services/ai/qdrant/pom.xml new file mode 100644 index 000000000..5392a1897 --- /dev/null +++ b/system-x/services/ai/qdrant/pom.xml @@ -0,0 +1,18 @@ + + + + system-x-ai + software.tnb + 1.0-SNAPSHOT + + 4.0.0 + + system-x-qdrant + 1.0-SNAPSHOT + TNB :: System-X :: Services :: Qdrant + + + + diff --git a/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/resource/local/LocalQdrant.java b/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/resource/local/LocalQdrant.java new file mode 100644 index 000000000..fecb85e98 --- /dev/null +++ b/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/resource/local/LocalQdrant.java @@ -0,0 +1,35 @@ +package software.tnb.qdrant.resource.local; + +import software.tnb.common.deployment.ContainerDeployable; +import software.tnb.qdrant.service.Qdrant; + +import com.google.auto.service.AutoService; + +@AutoService(Qdrant.class) +public class LocalQdrant extends Qdrant implements ContainerDeployable { + + private final QdrantContainer container = new QdrantContainer(defaultImage(), PORT); + + @Override + public String host() { + return container.getHost(); + } + + @Override + public int port() { + return container.getMappedPort(PORT); + } + + @Override + public void openResources() { + } + + @Override + public void closeResources() { + } + + @Override + public QdrantContainer container() { + return container; + } +} diff --git a/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/resource/local/QdrantContainer.java b/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/resource/local/QdrantContainer.java new file mode 100644 index 000000000..8c3e74952 --- /dev/null +++ b/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/resource/local/QdrantContainer.java @@ -0,0 +1,13 @@ +package software.tnb.qdrant.resource.local; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +public class QdrantContainer extends GenericContainer { + + public QdrantContainer(String image, int port) { + super(image); + this.withExposedPorts(port); + this.waitingFor(Wait.forLogMessage(".*starting in Actix runtime.*", 1)); + } +} diff --git a/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/resource/openshift/OpenshiftQdrant.java b/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/resource/openshift/OpenshiftQdrant.java new file mode 100644 index 000000000..79814e2f1 --- /dev/null +++ b/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/resource/openshift/OpenshiftQdrant.java @@ -0,0 +1,176 @@ +package software.tnb.qdrant.resource.openshift; + +import software.tnb.common.config.OpenshiftConfiguration; +import software.tnb.common.deployment.OpenshiftDeployable; +import software.tnb.common.deployment.WithName; +import software.tnb.common.openshift.OpenshiftClient; +import software.tnb.common.utils.NetworkUtils; +import software.tnb.common.utils.WaitUtils; +import software.tnb.common.utils.waiter.Waiter; +import software.tnb.qdrant.service.Qdrant; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.auto.service.AutoService; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; + +import io.fabric8.kubernetes.api.model.ContainerPort; +import io.fabric8.kubernetes.api.model.ContainerPortBuilder; +import io.fabric8.kubernetes.api.model.IntOrString; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.ServiceBuilder; +import io.fabric8.kubernetes.api.model.Volume; +import io.fabric8.kubernetes.api.model.VolumeBuilder; +import io.fabric8.kubernetes.api.model.VolumeMount; +import io.fabric8.kubernetes.api.model.VolumeMountBuilder; +import io.fabric8.kubernetes.client.PortForward; +import io.fabric8.openshift.api.model.RouteBuilder; + +@AutoService(Qdrant.class) +public class OpenshiftQdrant extends Qdrant implements OpenshiftDeployable, WithName { + private static final Logger LOG = LoggerFactory.getLogger(OpenshiftQdrant.class); + + private PortForward portForward; + private int localPort; + + @Override + public void undeploy() { + OpenshiftClient.get().apps().deployments().withName(name()).delete(); + OpenshiftClient.get().services().withLabel(OpenshiftConfiguration.openshiftDeploymentLabel(), name()).delete(); + OpenshiftClient.get().routes().withLabel(OpenshiftConfiguration.openshiftDeploymentLabel(), name()).delete(); + WaitUtils.waitFor(new Waiter(() -> servicePod() == null, "Waiting until the pod is removed")); + } + + @Override + public void openResources() { + localPort = NetworkUtils.getFreePort(); + portForward = OpenshiftClient.get().services().withName(name()).portForward(PORT, localPort); + } + + @Override + public void closeResources() { + NetworkUtils.releasePort(localPort); + if (portForward != null) { + try { + portForward.close(); + } catch (Exception e) { + LOG.warn("Unable to close Qdrant port forward", e); + } + } + } + + @Override + public void create() { + List ports = new LinkedList<>(); + ports.add(new ContainerPortBuilder() + .withName("http") + .withProtocol("TCP") + .withContainerPort(PORT) + .build()); + + List volumes = new LinkedList<>(); + volumes.add(new VolumeBuilder() + .withName("qdrant-storage") + .withNewEmptyDir() + .endEmptyDir() + .build()); + volumes.add(new VolumeBuilder() + .withName("qdrant-snapshots") + .withNewEmptyDir() + .endEmptyDir() + .build()); + + List volumeMounts = new LinkedList<>(); + volumeMounts.add(new VolumeMountBuilder() + .withName("qdrant-storage") + .withMountPath("/qdrant/storage") + .build()); + volumeMounts.add(new VolumeMountBuilder() + .withName("qdrant-snapshots") + .withMountPath("/qdrant/snapshots") + .build()); + + // @formatter:off + OpenshiftClient.get().createDeployment(Map.of( + "name", name(), + "image", image(), + "ports", ports, + "volumes", volumes, + "volumeMounts", volumeMounts + )); + + OpenshiftClient.get().services().resource(new ServiceBuilder() + .editOrNewMetadata() + .withName(name()) + .addToLabels(OpenshiftConfiguration.openshiftDeploymentLabel(), name()) + .endMetadata() + .editOrNewSpec() + .addToSelector(OpenshiftConfiguration.openshiftDeploymentLabel(), name()) + .addNewPort() + .withName("http") + .withProtocol("TCP") + .withPort(PORT) + .withTargetPort(new IntOrString(PORT)) + .endPort() + .endSpec() + .build() + ).serverSideApply(); + + OpenshiftClient.get().routes().resource(new RouteBuilder() + .editOrNewMetadata() + .withName(name()) + .addToLabels(OpenshiftConfiguration.openshiftDeploymentLabel(), name()) + .endMetadata() + .editOrNewSpec() + .withNewTo() + .withKind("Service") + .withName(name()) + .endTo() + .withNewPort() + .withNewTargetPort(PORT) + .endPort() + .withNewTls() + .withTermination("edge") + .withInsecureEdgeTerminationPolicy("Redirect") + .endTls() + .endSpec() + .build() + ).serverSideApply(); + // @formatter:on + } + + @Override + public boolean isDeployed() { + return WithName.super.isDeployed(); + } + + @Override + public Predicate podSelector() { + return WithName.super.podSelector(); + } + + @Override + public String host() { + return "localhost"; + } + + @Override + public int port() { + return localPort; + } + + @Override + public String getLogs() { + return OpenshiftDeployable.super.getLogs(); + } + + @Override + public String name() { + return "qdrant"; + } +} diff --git a/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/service/Qdrant.java b/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/service/Qdrant.java new file mode 100644 index 000000000..a80a77770 --- /dev/null +++ b/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/service/Qdrant.java @@ -0,0 +1,32 @@ +package software.tnb.qdrant.service; + +import software.tnb.common.account.NoAccount; +import software.tnb.common.client.NoClient; +import software.tnb.common.deployment.WithDockerImage; +import software.tnb.common.service.Service; +import software.tnb.qdrant.validation.QdrantValidation; + +public abstract class Qdrant extends Service implements WithDockerImage { + + protected static final int PORT = 6333; + + public abstract String host(); + + public abstract int port(); + + public String url() { + return String.format("http://%s:%d", host(), port()); + } + + public QdrantValidation validation() { + if (validation == null) { + validation = new QdrantValidation(url()); + } + return validation; + } + + @Override + public String defaultImage() { + return "quay.io/fuse_qe/qdrant:1.16.3"; + } +} diff --git a/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/validation/QdrantValidation.java b/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/validation/QdrantValidation.java new file mode 100644 index 000000000..d46d61209 --- /dev/null +++ b/system-x/services/ai/qdrant/src/main/java/software/tnb/qdrant/validation/QdrantValidation.java @@ -0,0 +1,51 @@ +package software.tnb.qdrant.validation; + +import software.tnb.common.utils.HTTPUtils; +import software.tnb.common.validation.Validation; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import okhttp3.MediaType; +import okhttp3.RequestBody; + +public class QdrantValidation implements Validation { + + private static final Logger LOG = LoggerFactory.getLogger(QdrantValidation.class); + private static final MediaType JSON = MediaType.parse("application/json"); + + private final String baseUrl; + + public QdrantValidation(String baseUrl) { + this.baseUrl = baseUrl; + } + + public String createCollection(String name, int vectorSize, String distance) { + LOG.debug("Creating collection '{}' with vectorSize={} and distance={}", name, vectorSize, distance); + String body = String.format("{\"vectors\":{\"size\":%d,\"distance\":\"%s\"}}", vectorSize, distance); + HTTPUtils.Response response = HTTPUtils.getInstance() + .put(baseUrl + "/collections/" + name, RequestBody.create(body, JSON)); + return response.getBody(); + } + + public String deleteCollection(String name) { + LOG.debug("Deleting collection '{}'", name); + HTTPUtils.getInstance().delete(baseUrl + "/collections/" + name); + return ""; + } + + public String upsert(String collectionName, String pointsJson) { + LOG.debug("Upserting points into collection '{}'", collectionName); + String body = String.format("{\"points\":%s}", pointsJson); + HTTPUtils.Response response = HTTPUtils.getInstance() + .put(baseUrl + "/collections/" + collectionName + "/points", RequestBody.create(body, JSON)); + return response.getBody(); + } + + public String query(String collectionName, String queryJson) { + LOG.debug("Querying collection '{}'", collectionName); + HTTPUtils.Response response = HTTPUtils.getInstance() + .post(baseUrl + "/collections/" + collectionName + "/points/query", RequestBody.create(queryJson, JSON)); + return response.getBody(); + } +} diff --git a/system-x/services/all/pom.xml b/system-x/services/all/pom.xml index 7c583869e..e41166c66 100644 --- a/system-x/services/all/pom.xml +++ b/system-x/services/all/pom.xml @@ -137,6 +137,14 @@ software.tnb system-x-docling + + software.tnb + system-x-ollama + + + software.tnb + system-x-qdrant + software.tnb system-x-elasticsearch @@ -301,10 +309,6 @@ software.tnb system-x-observability - - software.tnb - system-x-ollama - software.tnb system-x-opensearch