diff --git a/docker/.env b/docker/.env index 169dc81..ded2109 100644 --- a/docker/.env +++ b/docker/.env @@ -32,12 +32,12 @@ REG_API_APP_PROPERTIES_FILE=./default-config/application.properties # -------------------------------------------------------------------- # Docker image of the Registry Loader -REG_LOADER_IMAGE=nasapds/registry-loader:latest +REG_LOADER_IMAGE=nasapds/registry-loader:geo # -------------------------------------------------------------------- # Registry Sweepers # -------------------------------------------------------------------- -REG_SWEEPERS_IMAGE=nasapds/registry-sweepers:latest +REG_SWEEPERS_IMAGE=nasapds/registry-sweepers:1.5.0 PROV_CREDENTIALS={"admin":"admin"} # -------------------------------------------------------------------- diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 9528d0e..02f7752 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -74,7 +74,8 @@ services: - LOGLEVEL=INFO - DEV_MODE=1 - MULTITENANCY_NODE_ID=geo - command: ["/usr/local/bin/sweepers_driver.py", "--legacy-sync"] + #command: ["/usr/local/bin/sweepers_driver.py", "--legacy-sync"] + command: ["/usr/local/bin/sweepers_driver.py"] networks: - pds depends_on: @@ -109,6 +110,8 @@ services: volumes: - ./scripts/elasticsearch-init.sh:/usr/local/bin/elasticsearch-init.sh - ./scripts/aliases:/usr/local/bin/aliases + - ./scripts/pipelines:/usr/local/bin/pipelines + - ./scripts/mappings:/usr/local/bin/mappings - ./default-config/es-auth.cfg:/etc/es-auth.cfg - ./default-config/local_registry.xml:/etc/local_registry.xml networks: diff --git a/docker/scripts/aliases/alias_registry.json b/docker/scripts/aliases/alias_registry.json index 3922c71..479c85c 100644 --- a/docker/scripts/aliases/alias_registry.json +++ b/docker/scripts/aliases/alias_registry.json @@ -2,7 +2,7 @@ "actions": [ { "add": { - "index": "*-registry", + "index": "geo-registry", "alias": "registry" } } diff --git a/docker/scripts/elasticsearch-init.sh b/docker/scripts/elasticsearch-init.sh old mode 100644 new mode 100755 index ef8c3e4..f0bab5c --- a/docker/scripts/elasticsearch-init.sh +++ b/docker/scripts/elasticsearch-init.sh @@ -34,6 +34,9 @@ # This script is used to initialize Elasticsearch. # ------------------------------------------------- +# Get the directory where this script is located +SCRIPT_DIR=$(dirname "$0") + # Check if the ES_URL environment variable is set if [ -z "$ES_URL" ]; then echo "Error: 'ES_URL' (Elasticsearch URL) environment variable is not set. Use docker's -e option." 1>&2 @@ -52,3 +55,21 @@ registry-manager create-registry -registry file:///etc/local_registry.xml -auth echo "Create aliases, until registry-manager does it" curl -X POST -H "Content-Type: application/json" -d @/usr/local/bin/aliases/alias_registry.json -u admin:admin --insecure https://elasticsearch:9200/_aliases +# build pipelines from source files +echo "Building pipeline JSON files from Painless sources..." +bash "$SCRIPT_DIR/pipelines/build_json.sh" \ + "$SCRIPT_DIR/pipelines/bbox_to_polygon.painless" \ + /tmp/bbox_to_polygon.json \ + "Builds a geo_shape polygon from bbox fields" + +# in test we have a single registry index, +# in production this needs to be repeated over all registry indices +#curl -X PUT -u admin:admin --insecure "https://elasticsearch:9200/geo-registry/_mapping" -H 'Content-Type: application/json' -d "@$SCRIPT_DIR/mappings/bbox_polygon.json" + +# create pipelines for geographical objects +echo "Creating bbox_to_polygon pipeline..." +curl -X PUT -u admin:admin --insecure "https://elasticsearch:9200/_ingest/pipeline/bbox_to_polygon" -H 'Content-Type: application/json' -d @/tmp/bbox_to_polygon.json + +# associate the pipeline to the registry index +curl -X PUT -u admin:admin --insecure "https://elasticsearch:9200/geo-registry/_settings" -H 'Content-Type: application/json' -d '{"index.default_pipeline": "bbox_to_polygon"}' + diff --git a/docker/scripts/mappings/bbox_polygon.json b/docker/scripts/mappings/bbox_polygon.json new file mode 100644 index 0000000..eb6eefb --- /dev/null +++ b/docker/scripts/mappings/bbox_polygon.json @@ -0,0 +1,5 @@ +{ + "properties": { + "bbox_polygon": { "type": "geo_shape" } + } +} \ No newline at end of file diff --git a/docker/scripts/pipelines/bbox_to_polygon.painless b/docker/scripts/pipelines/bbox_to_polygon.painless new file mode 100644 index 0000000..84949e2 --- /dev/null +++ b/docker/scripts/pipelines/bbox_to_polygon.painless @@ -0,0 +1,29 @@ + +double getBoundary(def ctx, String field) { + + if (!ctx.containsKey(field)) { + return Double.NaN; + } + + return Double.parseDouble(ctx[field][0].toString()); +} + +def east = getBoundary(ctx, 'cart:Bounding_Coordinates/cart:east_bounding_coordinate'); +def west = getBoundary(ctx, 'cart:Bounding_Coordinates/cart:west_bounding_coordinate'); +def north = getBoundary(ctx, 'cart:Bounding_Coordinates/cart:north_bounding_coordinate'); +def south = getBoundary(ctx, 'cart:Bounding_Coordinates/cart:south_bounding_coordinate'); + + +if (!Double.isNaN(east) && !Double.isNaN(west) && !Double.isNaN(north) && !Double.isNaN(south)) { + ctx.bbox_polygon = [ + 'type': 'Polygon', + 'coordinates': [[ + [west, south], + [east, south], + [east, north], + [west, north], + [west, south] + ]] + ]; +} + diff --git a/docker/scripts/pipelines/build_json.sh b/docker/scripts/pipelines/build_json.sh new file mode 100755 index 0000000..6432847 --- /dev/null +++ b/docker/scripts/pipelines/build_json.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Build a JSON pipeline file from a Painless source file. +# This script properly escapes JSON special characters. + +set -e + +if [ $# -ne 3 ]; then + echo "Usage: build_json.sh " >&2 + exit 1 +fi + +painless_file="$1" +json_file="$2" +description="$3" + +# Read painless source and join into single line (stripping empty lines) +source="" +while IFS= read -r line || [ -n "$line" ]; do + trimmed_line="${line#"${line%%[![:space:]]*}"}" + trimmed_line="${trimmed_line%"${trimmed_line##*[![:space:]]}"}" + if [ -n "$trimmed_line" ]; then + if [ -z "$source" ]; then + source="$trimmed_line" + else + source="$source $trimmed_line" + fi + fi +done < "$painless_file" + +# Escape special characters for JSON using sed +# Escape backslash, double quote, forward slash, AND & +escaped_source=$(printf '%s' "$source" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\//\\\//g; s/&/\\&/g') + +# Write JSON file (avoid variable expansion in heredoc) +cat > "$json_file" <<'JSONEOF' +{ + "description": "DESCRIPTION_PLACEHOLDER", + "processors": [ + { + "script": { + "lang": "painless", + "source": "SOURCE_PLACEHOLDER" + } + } + ] +} +JSONEOF + +# Replace placeholders with actual values +sed "s/DESCRIPTION_PLACEHOLDER/$description/g; s/SOURCE_PLACEHOLDER/$escaped_source/g" "$json_file" > "${json_file}.tmp" +mv "${json_file}.tmp" "$json_file" + +echo "Built: $json_file" +