Add Geo Shape in Registry with ingest pipelines

docker/.env

@@ -32,12 +32,12 @@ REG_API_APP_PROPERTIES_FILE=./default-config/application.properties
 # --------------------------------------------------------------------
 
 # Docker image of the Registry Loader
-REG_LOADER_IMAGE=nasapds/registry-loader:latest
+REG_LOADER_IMAGE=nasapds/registry-loader:geo
 
 # --------------------------------------------------------------------
 # Registry Sweepers
 # --------------------------------------------------------------------
-REG_SWEEPERS_IMAGE=nasapds/registry-sweepers:latest
+REG_SWEEPERS_IMAGE=nasapds/registry-sweepers:1.5.0
 PROV_CREDENTIALS={"admin":"admin"}
 
 # --------------------------------------------------------------------

docker/docker-compose.yml

@@ -74,7 +74,8 @@ services:
       - LOGLEVEL=INFO
       - DEV_MODE=1
       - MULTITENANCY_NODE_ID=geo
-    command: ["/usr/local/bin/sweepers_driver.py", "--legacy-sync"]
+    #command: ["/usr/local/bin/sweepers_driver.py", "--legacy-sync"]
+    command: ["/usr/local/bin/sweepers_driver.py"]
     networks:
       - pds
     depends_on:
@@ -109,6 +110,8 @@ services:
     volumes:
       - ./scripts/elasticsearch-init.sh:/usr/local/bin/elasticsearch-init.sh
       - ./scripts/aliases:/usr/local/bin/aliases
+      - ./scripts/pipelines:/usr/local/bin/pipelines
+      - ./scripts/mappings:/usr/local/bin/mappings
       - ./default-config/es-auth.cfg:/etc/es-auth.cfg
       - ./default-config/local_registry.xml:/etc/local_registry.xml
     networks:

docker/scripts/aliases/alias_registry.json

@@ -2,7 +2,7 @@
   "actions": [
     {
       "add": {
-        "index": "*-registry",
+        "index": "geo-registry",
         "alias": "registry"
       }
     }

docker/scripts/elasticsearch-init.sh

@@ -34,6 +34,9 @@
 # This script is used to initialize Elasticsearch.
 # -------------------------------------------------
 
+# Get the directory where this script is located
+SCRIPT_DIR=$(dirname "$0")
+
 # Check if the ES_URL environment variable is set
 if [ -z "$ES_URL" ]; then
     echo "Error: 'ES_URL' (Elasticsearch URL) environment variable is not set. Use docker's -e option." 1>&2
@@ -52,3 +55,21 @@ registry-manager create-registry -registry file:///etc/local_registry.xml  -auth
 echo "Create aliases, until registry-manager does it"
 curl -X POST  -H "Content-Type: application/json" -d @/usr/local/bin/aliases/alias_registry.json -u admin:admin --insecure https://elasticsearch:9200/_aliases
 
+# build pipelines from source files
+echo "Building pipeline JSON files from Painless sources..."
+bash "$SCRIPT_DIR/pipelines/build_json.sh" \
+  "$SCRIPT_DIR/pipelines/bbox_to_polygon.painless" \
+  /tmp/bbox_to_polygon.json \
+  "Builds a geo_shape polygon from bbox fields"
+
+# in test we have a single registry index,
+# in production this needs to be repeated over all registry indices
+#curl -X PUT -u admin:admin --insecure "https://elasticsearch:9200/geo-registry/_mapping" -H 'Content-Type: application/json' -d "@$SCRIPT_DIR/mappings/bbox_polygon.json"
+
+# create pipelines for geographical objects
+echo "Creating bbox_to_polygon pipeline..."
+curl -X PUT -u admin:admin --insecure "https://elasticsearch:9200/_ingest/pipeline/bbox_to_polygon" -H 'Content-Type: application/json' -d @/tmp/bbox_to_polygon.json
+
+# associate the pipeline to the registry index
+curl -X PUT -u admin:admin --insecure "https://elasticsearch:9200/geo-registry/_settings" -H 'Content-Type: application/json' -d '{"index.default_pipeline": "bbox_to_polygon"}'
+

docker/scripts/mappings/bbox_polygon.json

@@ -0,0 +1,5 @@
+{
+  "properties": {
+    "bbox_polygon": { "type": "geo_shape" }
+  }
+}

docker/scripts/pipelines/bbox_to_polygon.painless

@@ -0,0 +1,29 @@
+
+double getBoundary(def ctx, String field) {
+
+    if (!ctx.containsKey(field)) {
+        return Double.NaN;
+    }
+
+    return Double.parseDouble(ctx[field][0].toString());
+}
+
+def east = getBoundary(ctx, 'cart:Bounding_Coordinates/cart:east_bounding_coordinate');
+def west = getBoundary(ctx, 'cart:Bounding_Coordinates/cart:west_bounding_coordinate');
+def north = getBoundary(ctx, 'cart:Bounding_Coordinates/cart:north_bounding_coordinate');
+def south = getBoundary(ctx, 'cart:Bounding_Coordinates/cart:south_bounding_coordinate');
+
+
+if (!Double.isNaN(east) && !Double.isNaN(west) &&  !Double.isNaN(north) && !Double.isNaN(south)) {
+    ctx.bbox_polygon = [
+        'type': 'Polygon',
+        'coordinates': [[
+            [west, south],
+            [east, south],
+            [east, north],
+            [west, north],
+            [west, south]
+        ]]
+    ];
+}
+

docker/scripts/pipelines/build_json.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Build a JSON pipeline file from a Painless source file.
+# This script properly escapes JSON special characters.
+
+set -e
+
+if [ $# -ne 3 ]; then
+    echo "Usage: build_json.sh <painless_file> <json_file> <description>" >&2
+    exit 1
+fi
+
+painless_file="$1"
+json_file="$2"
+description="$3"
+
+# Read painless source and join into single line (stripping empty lines)
+source=""
+while IFS= read -r line || [ -n "$line" ]; do
+    trimmed_line="${line#"${line%%[![:space:]]*}"}"
+    trimmed_line="${trimmed_line%"${trimmed_line##*[![:space:]]}"}"
+    if [ -n "$trimmed_line" ]; then
+        if [ -z "$source" ]; then
+            source="$trimmed_line"
+        else
+            source="$source $trimmed_line"
+        fi
+    fi
+done < "$painless_file"
+
+# Escape special characters for JSON using sed
+# Escape backslash, double quote, forward slash, AND &
+escaped_source=$(printf '%s' "$source" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\//\\\//g; s/&/\\&/g')
+
+# Write JSON file (avoid variable expansion in heredoc)
+cat > "$json_file" <<'JSONEOF'
+{
+  "description": "DESCRIPTION_PLACEHOLDER",
+  "processors": [
+    {
+      "script": {
+        "lang": "painless",
+        "source": "SOURCE_PLACEHOLDER"
+      }
+    }
+  ]
+}
+JSONEOF
+
+# Replace placeholders with actual values
+sed "s/DESCRIPTION_PLACEHOLDER/$description/g; s/SOURCE_PLACEHOLDER/$escaped_source/g" "$json_file" > "${json_file}.tmp"
+mv "${json_file}.tmp" "$json_file"
+
+echo "Built: $json_file"
+